aboutsummaryrefslogtreecommitdiffstats
path: root/library
diff options
context:
space:
mode:
authorMaxim Yurchuk <maxim-yurchuk@ydb.tech>2024-11-20 17:37:57 +0000
committerGitHub <noreply@github.com>2024-11-20 17:37:57 +0000
commitf76323e9b295c15751e51e3443aa47a36bee8023 (patch)
tree4113c8cad473a33e0f746966e0cf087252fa1d7a /library
parent753ecb8d410a4cb459c26f3a0082fb2d1724fe63 (diff)
parenta7b9a6afea2a9d7a7bfac4c5eb4c1a8e60adb9e6 (diff)
downloadydb-f76323e9b295c15751e51e3443aa47a36bee8023.tar.gz
Merge pull request #11788 from ydb-platform/mergelibs-241120-1113
Library import 241120-1113
Diffstat (limited to 'library')
-rw-r--r--library/cpp/lwtrace/rwspinlock.h24
-rw-r--r--library/cpp/netliba/v6/udp_address.cpp2
-rw-r--r--library/cpp/pop_count/benchmark/main.cpp82
-rw-r--r--library/cpp/pop_count/benchmark/ya.make12
-rw-r--r--library/cpp/pop_count/popcount.cpp30
-rw-r--r--library/cpp/pop_count/popcount.h105
-rw-r--r--library/cpp/pop_count/popcount_ut.cpp70
-rw-r--r--library/cpp/pop_count/ut/ya.make7
-rw-r--r--library/cpp/pop_count/ya.make12
-rw-r--r--library/cpp/string_utils/ztstrbuf/ztstrbuf_ut.cpp2
-rw-r--r--library/cpp/svnversion/ya.make2
-rw-r--r--library/cpp/testing/common/env.cpp4
-rw-r--r--library/cpp/testing/mock_server/ut/server_ut.cpp40
-rw-r--r--library/cpp/testing/mock_server/ut/ya.make11
-rw-r--r--library/cpp/tld/tlds-alpha-by-domain.txt2
-rw-r--r--library/cpp/yt/misc/cast-inl.h13
-rw-r--r--library/cpp/yt/misc/cast.h2
-rw-r--r--library/cpp/yt/misc/unittests/cast_ut.cpp49
-rw-r--r--library/cpp/yt/string/format-inl.h236
-rw-r--r--library/cpp/yt/string/format_analyser.h42
-rw-r--r--library/cpp/yt/string/format_arg.h2
-rw-r--r--library/cpp/yt/string/format_string-inl.h16
-rw-r--r--library/cpp/yt/string/format_string.h4
-rw-r--r--library/cpp/yt/string/unittests/format_ut.cpp5
-rw-r--r--library/python/testing/yatest_common/yatest/common/runtime.py2
25 files changed, 336 insertions, 440 deletions
diff --git a/library/cpp/lwtrace/rwspinlock.h b/library/cpp/lwtrace/rwspinlock.h
index 5e4608e068..1effd5fe0e 100644
--- a/library/cpp/lwtrace/rwspinlock.h
+++ b/library/cpp/lwtrace/rwspinlock.h
@@ -1,6 +1,6 @@
#pragma once
-#include <library/cpp/deprecated/atomic/atomic.h>
+#include <atomic>
#include <util/system/spinlock.h>
@@ -27,16 +27,16 @@
// * writer can release lock (State = 0: -> READING)
struct TRWSpinLock {
- TAtomic State; // must be initialized by 'TRWSpinLock myLock = {0};' construction
+ std::atomic_signed_lock_free State;
void Init() noexcept {
- State = 0;
+ State.store(0, std::memory_order_relaxed);
}
void AcquireRead() noexcept {
while (true) {
- TAtomic a = AtomicGet(State);
- if ((a & 1) == 0 && AtomicCas(&State, a + 2, a)) {
+ std::atomic_signed_lock_free::value_type a = State.load(std::memory_order_acquire);
+ if ((a & 1) == 0 && State.compare_exchange_strong(a, a + 2, std::memory_order_acquire)) {
break;
}
SpinLockPause();
@@ -44,25 +44,29 @@ struct TRWSpinLock {
}
void ReleaseRead() noexcept {
- AtomicAdd(State, -2);
+ State.fetch_add(-2, std::memory_order_release);
}
void AcquireWrite() noexcept {
while (true) {
- TAtomic a = AtomicGet(State);
- if ((a & 1) == 0 && AtomicCas(&State, a + 1, a)) {
+ std::atomic_signed_lock_free::value_type a = State.load(std::memory_order_acquire);
+ if ((a & 1) == 0 && State.compare_exchange_strong(a, a + 1, std::memory_order_acquire)) {
break;
}
SpinLockPause();
}
- while (!AtomicCas(&State, TAtomicBase(-1), 1)) {
+ while (true) {
+ std::atomic_signed_lock_free::value_type a = 1;
+ if (State.compare_exchange_strong(a, -1, std::memory_order_acquire)) {
+ break;
+ }
SpinLockPause();
}
}
void ReleaseWrite() noexcept {
- AtomicSet(State, 0);
+ State.store(0, std::memory_order_release);
}
};
diff --git a/library/cpp/netliba/v6/udp_address.cpp b/library/cpp/netliba/v6/udp_address.cpp
index ba5fd6f45b..be823e2dfa 100644
--- a/library/cpp/netliba/v6/udp_address.cpp
+++ b/library/cpp/netliba/v6/udp_address.cpp
@@ -196,7 +196,7 @@ namespace NNetliba {
continue;
}
if (ptr->Mtu < 1280) {
- fprintf(stderr, "WARNING: MTU %d is less then ipv6 minimum", ptr->Mtu);
+ fprintf(stderr, "WARNING: MTU %lu is less then ipv6 minimum", ptr->Mtu);
}
for (IP_ADAPTER_UNICAST_ADDRESS* addr = ptr->FirstUnicastAddress; addr; addr = addr->Next) {
sockaddr* x = (sockaddr*)addr->Address.lpSockaddr;
diff --git a/library/cpp/pop_count/benchmark/main.cpp b/library/cpp/pop_count/benchmark/main.cpp
deleted file mode 100644
index 1af2d157b6..0000000000
--- a/library/cpp/pop_count/benchmark/main.cpp
+++ /dev/null
@@ -1,82 +0,0 @@
-#include <util/stream/output.h>
-#include <util/datetime/cputimer.h>
-#include <util/system/type_name.h>
-
-#include <library/cpp/pop_count/popcount.h>
-#include <library/cpp/testing/benchmark/bench.h>
-
-#include <bit>
-
-template <class F, class I>
-inline void DoRun(F&& f, I&& i) {
- const ui64 n = i.Iterations();
-
- for (ui64 j = 0; j < n; ++j) {
- Y_DO_NOT_OPTIMIZE_AWAY(f(j * (ui64)123456 + (ui64)1));
- }
-}
-
-Y_CPU_BENCHMARK(PopCount_8, iface) {
- DoRun([](ui8 x) {
- return PopCount<ui8>(x);
- },
- iface);
-}
-
-Y_CPU_BENCHMARK(std_popcount_8, iface) {
- DoRun([](ui8 x) {
- return std::popcount<ui8>(x);
- },
- iface);
-}
-
-Y_CPU_BENCHMARK(PopCount_16, iface) {
- DoRun([](ui16 x) {
- return PopCount<ui16>(x);
- },
- iface);
-}
-
-Y_CPU_BENCHMARK(std_popcount_16, iface) {
- DoRun([](ui16 x) {
- return std::popcount<ui16>(x);
- },
- iface);
-}
-
-Y_CPU_BENCHMARK(PopCount_32, iface) {
- DoRun([](ui32 x) {
- return PopCount<ui32>(x);
- },
- iface);
-}
-
-Y_CPU_BENCHMARK(std_popcount_32, iface) {
- DoRun([](ui32 x) {
- return std::popcount<ui32>(x);
- },
- iface);
-}
-
-Y_CPU_BENCHMARK(PopCount_64, iface) {
- DoRun([](ui64 x) {
- return PopCount<ui64>(x);
- },
- iface);
-}
-
-Y_CPU_BENCHMARK(std_popcount_64, iface) {
- DoRun([](ui64 x) {
- return std::popcount<ui64>(x);
- },
- iface);
-}
-
-#if !defined(_MSC_VER)
-Y_CPU_BENCHMARK(BUILTIN_64, iface) {
- DoRun([](ui64 x) {
- return __builtin_popcountll(x);
- },
- iface);
-}
-#endif
diff --git a/library/cpp/pop_count/benchmark/ya.make b/library/cpp/pop_count/benchmark/ya.make
deleted file mode 100644
index e7ea9418db..0000000000
--- a/library/cpp/pop_count/benchmark/ya.make
+++ /dev/null
@@ -1,12 +0,0 @@
-Y_BENCHMARK()
-
-PEERDIR(
- util/draft
- library/cpp/pop_count
-)
-
-SRCS(
- main.cpp
-)
-
-END()
diff --git a/library/cpp/pop_count/popcount.cpp b/library/cpp/pop_count/popcount.cpp
deleted file mode 100644
index 49276424be..0000000000
--- a/library/cpp/pop_count/popcount.cpp
+++ /dev/null
@@ -1,30 +0,0 @@
-#include "popcount.h"
-
-#include <util/system/defaults.h>
-#include <util/system/yassert.h>
-
-#include <string.h>
-
-static const ui8 PopCountLUT8Impl[1 << 8] = {
-#define B2(n) n, n + 1, n + 1, n + 2
-#define B4(n) B2(n), B2(n + 1), B2(n + 1), B2(n + 2)
-#define B6(n) B4(n), B4(n + 1), B4(n + 1), B4(n + 2)
- B6(0), B6(1), B6(1), B6(2)};
-
-ui8 const* PopCountLUT8 = PopCountLUT8Impl;
-
-#if !defined(_MSC_VER)
-//ICE here for msvc
-
-static const ui8 PopCountLUT16Impl[1 << 16] = {
-#define B2(n) n, n + 1, n + 1, n + 2
-#define B4(n) B2(n), B2(n + 1), B2(n + 1), B2(n + 2)
-#define B6(n) B4(n), B4(n + 1), B4(n + 1), B4(n + 2)
-#define B8(n) B6(n), B6(n + 1), B6(n + 1), B6(n + 2)
-#define B10(n) B8(n), B8(n + 1), B8(n + 1), B8(n + 2)
-#define B12(n) B10(n), B10(n + 1), B10(n + 1), B10(n + 2)
-#define B14(n) B12(n), B12(n + 1), B12(n + 1), B12(n + 2)
- B14(0), B14(1), B14(1), B14(2)};
-
-ui8 const* PopCountLUT16 = PopCountLUT16Impl;
-#endif
diff --git a/library/cpp/pop_count/popcount.h b/library/cpp/pop_count/popcount.h
deleted file mode 100644
index 3d67737ed2..0000000000
--- a/library/cpp/pop_count/popcount.h
+++ /dev/null
@@ -1,105 +0,0 @@
-#pragma once
-
-#include <util/generic/typelist.h>
-#include <util/system/cpu_id.h>
-#include <util/system/defaults.h>
-#include <util/system/hi_lo.h>
-#include <util/system/platform.h>
-
-#if defined(_MSC_VER)
-#include <intrin.h>
-#endif
-
-static inline ui32 PopCountImpl(ui8 n) {
-#if defined(_ppc64_)
- ui32 r;
- __asm__("popcntb %0, %1"
- : "=r"(r)
- : "r"(n)
- :);
- return r;
-#else
- extern ui8 const* PopCountLUT8;
- return PopCountLUT8[n];
-#endif
-}
-
-static inline ui32 PopCountImpl(ui16 n) {
-#if defined(_MSC_VER)
- return __popcnt16(n);
-#else
- extern ui8 const* PopCountLUT16;
- return PopCountLUT16[n];
-#endif
-}
-
-static inline ui32 PopCountImpl(ui32 n) {
-#if defined(_MSC_VER)
- return __popcnt(n);
-#else
-#if defined(_x86_64_)
- if (NX86::CachedHavePOPCNT()) {
- ui32 r;
-
- __asm__("popcnt %1, %0;"
- : "=r"(r)
- : "r"(n)
- :);
-
- return r;
- }
-#else
-#if defined(_ppc64_)
- ui32 r;
-
- __asm__("popcntw %0, %1"
- : "=r"(r)
- : "r"(n)
- :);
-
- return r;
-#endif
-#endif
-
- return PopCountImpl((ui16)Lo16(n)) + PopCountImpl((ui16)Hi16(n));
-#endif
-}
-
-static inline ui32 PopCountImpl(ui64 n) {
-#if defined(_MSC_VER) && !defined(_i386_)
- return __popcnt64(n);
-#else
-#if defined(_x86_64_)
- if (NX86::CachedHavePOPCNT()) {
- ui64 r;
-
- __asm__("popcnt %1, %0;"
- : "=r"(r)
- : "r"(n)
- :);
-
- return r;
- }
-#else
-#if defined(_ppc64_)
- ui32 r;
-
- __asm__("popcntd %0, %1"
- : "=r"(r)
- : "r"(n)
- :);
-
- return r;
-#endif
-#endif
-
- return PopCountImpl((ui32)Lo32(n)) + PopCountImpl((ui32)Hi32(n));
-#endif
-}
-
-template <class T>
-static inline ui32 PopCount(T n) {
- using TCvt = TFixedWidthUnsignedInt<T>;
-
- return PopCountImpl((TCvt)n);
-}
diff --git a/library/cpp/pop_count/popcount_ut.cpp b/library/cpp/pop_count/popcount_ut.cpp
deleted file mode 100644
index 5cd6605411..0000000000
--- a/library/cpp/pop_count/popcount_ut.cpp
+++ /dev/null
@@ -1,70 +0,0 @@
-#include "popcount.h"
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/random/random.h>
-
-Y_UNIT_TEST_SUITE(TestPopCount) {
- template <class T>
- static inline ui32 SlowPopCount(T t) {
- ui32 ret = 0;
-
- while (t) {
- if (t & T(1)) {
- ++ret;
- }
-
- t = t >> 1;
- }
-
- return ret;
- }
-
- template <class T>
- static inline void Test() {
- for (size_t i = 0; i < 10000; ++i) {
- const T rndv = RandomNumber<T>();
-
- UNIT_ASSERT_VALUES_EQUAL(SlowPopCount(rndv), PopCount(rndv));
- }
- }
-
- Y_UNIT_TEST(Test8) {
- Test<ui8>();
- }
-
- Y_UNIT_TEST(Test16) {
- Test<ui16>();
- }
-
- Y_UNIT_TEST(Test32) {
- Test<ui32>();
- }
-
- Y_UNIT_TEST(Test64) {
- Test<ui64>();
- }
-
- Y_UNIT_TEST(TestPopCount) {
- UNIT_ASSERT_VALUES_EQUAL(PopCount(0), 0);
- UNIT_ASSERT_VALUES_EQUAL(PopCount(1), 1);
- UNIT_ASSERT_VALUES_EQUAL(PopCount(1 << 10), 1);
- UNIT_ASSERT_VALUES_EQUAL(PopCount((1 << 10) + 1), 2);
- UNIT_ASSERT_VALUES_EQUAL(PopCount(0xFFFF), 16);
- UNIT_ASSERT_VALUES_EQUAL(PopCount(0xFFFFFFFF), 32);
- UNIT_ASSERT_VALUES_EQUAL(PopCount(0x55555555), 16);
-
- UNIT_ASSERT_VALUES_EQUAL(0, PopCount(0ULL));
- UNIT_ASSERT_VALUES_EQUAL(1, PopCount(1ULL));
- UNIT_ASSERT_VALUES_EQUAL(16, PopCount(0xAAAAAAAAULL));
- UNIT_ASSERT_VALUES_EQUAL(32, PopCount(0xFFFFFFFFULL));
- UNIT_ASSERT_VALUES_EQUAL(32, PopCount(0xAAAAAAAAAAAAAAAAULL));
- UNIT_ASSERT_VALUES_EQUAL(64, PopCount(0xFFFFFFFFFFFFFFFFULL));
-
- ui64 v = 0;
-
- for (int i = 0; i < 64; v |= 1ULL << i, ++i) {
- UNIT_ASSERT_VALUES_EQUAL(i, PopCount(v));
- }
- }
-}
diff --git a/library/cpp/pop_count/ut/ya.make b/library/cpp/pop_count/ut/ya.make
deleted file mode 100644
index 179d47f064..0000000000
--- a/library/cpp/pop_count/ut/ya.make
+++ /dev/null
@@ -1,7 +0,0 @@
-UNITTEST_FOR(library/cpp/pop_count)
-
-SRCS(
- popcount_ut.cpp
-)
-
-END()
diff --git a/library/cpp/pop_count/ya.make b/library/cpp/pop_count/ya.make
deleted file mode 100644
index d858f3e87e..0000000000
--- a/library/cpp/pop_count/ya.make
+++ /dev/null
@@ -1,12 +0,0 @@
-LIBRARY()
-
-SRCS(
- popcount.cpp
-)
-
-END()
-
-RECURSE(
- benchmark
- ut
-)
diff --git a/library/cpp/string_utils/ztstrbuf/ztstrbuf_ut.cpp b/library/cpp/string_utils/ztstrbuf/ztstrbuf_ut.cpp
index d8215abdfa..b22d4bf216 100644
--- a/library/cpp/string_utils/ztstrbuf/ztstrbuf_ut.cpp
+++ b/library/cpp/string_utils/ztstrbuf/ztstrbuf_ut.cpp
@@ -12,7 +12,7 @@ Y_UNIT_TEST_SUITE(TZtStringBufTest) {
Y_UNIT_TEST(Constness) {
constexpr TZtStringBuf s0{"bar"};
static_assert(s0[0] == 'b');
- static_assert(s0[3] == '\0');
+ static_assert(s0.data()[s0.size()] == '\0');
static_assert(s0.data()[2] == 'r');
UNIT_ASSERT_VALUES_EQUAL(s0, TString{"bar"});
}
diff --git a/library/cpp/svnversion/ya.make b/library/cpp/svnversion/ya.make
index d028f45c94..27e9154644 100644
--- a/library/cpp/svnversion/ya.make
+++ b/library/cpp/svnversion/ya.make
@@ -6,7 +6,7 @@ SRCS(
)
IF (OPENSOURCE_PROJECT == "yt-cpp-sdk")
- PEERDIR(build/scripts/c_templates/)
+ PEERDIR(build/scripts/c_templates)
ENDIF()
END()
diff --git a/library/cpp/testing/common/env.cpp b/library/cpp/testing/common/env.cpp
index 13f3f186af..67e081c371 100644
--- a/library/cpp/testing/common/env.cpp
+++ b/library/cpp/testing/common/env.cpp
@@ -41,6 +41,10 @@ TString BinaryPath(TStringBuf path) {
}
TString GetArcadiaTestsData() {
+ if (GetEnv("USE_ATD_FROM_SNAPSHOT")) {
+ return ArcadiaSourceRoot() + "/atd_ro_snapshot";
+ }
+
TString atdRoot = NPrivate::GetTestEnv().ArcadiaTestsDataDir;
if (atdRoot) {
return atdRoot;
diff --git a/library/cpp/testing/mock_server/ut/server_ut.cpp b/library/cpp/testing/mock_server/ut/server_ut.cpp
deleted file mode 100644
index 74b3dcec05..0000000000
--- a/library/cpp/testing/mock_server/ut/server_ut.cpp
+++ /dev/null
@@ -1,40 +0,0 @@
-#include <library/cpp/testing/mock_server/server.h>
-
-#include <library/cpp/http/simple/http_client.h>
-#include <library/cpp/testing/unittest/registar.h>
-
-Y_UNIT_TEST_SUITE(Server) {
- int i;
-
- Y_UNIT_TEST(pong) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(port, []() { return new NMock::TPong; });
-
- TKeepAliveHttpClient cl("localhost", port);
- UNIT_ASSERT_VALUES_EQUAL(200, cl.DoGet("/ping"));
- UNIT_ASSERT_VALUES_EQUAL(404, cl.DoGet("/kek"));
- }
-
- Y_UNIT_TEST(custom) {
- class TCustomReplier: public TRequestReplier {
- public:
- bool DoReply(const TReplyParams& params) override {
- THttpResponse resp(HttpCodes::HTTP_OK);
- resp.SetContent("everithing is ok");
- resp.OutTo(params.Output);
-
- return true;
- }
- };
-
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(port, []() { return new TCustomReplier; });
-
- TKeepAliveHttpClient cl("localhost", port);
- TStringStream out;
- UNIT_ASSERT_VALUES_EQUAL(200, cl.DoGet("/foo", &out));
- UNIT_ASSERT_VALUES_EQUAL("everithing is ok", out.Str());
- }
-}
diff --git a/library/cpp/testing/mock_server/ut/ya.make b/library/cpp/testing/mock_server/ut/ya.make
deleted file mode 100644
index f4ee34458f..0000000000
--- a/library/cpp/testing/mock_server/ut/ya.make
+++ /dev/null
@@ -1,11 +0,0 @@
-UNITTEST_FOR(library/cpp/testing/mock_server)
-
-PEERDIR(
- library/cpp/http/simple
-)
-
-SRCS(
- server_ut.cpp
-)
-
-END()
diff --git a/library/cpp/tld/tlds-alpha-by-domain.txt b/library/cpp/tld/tlds-alpha-by-domain.txt
index 75edec6f55..dc6477cc73 100644
--- a/library/cpp/tld/tlds-alpha-by-domain.txt
+++ b/library/cpp/tld/tlds-alpha-by-domain.txt
@@ -1,4 +1,4 @@
-# Version 2024111300, Last Updated Wed Nov 13 07:07:01 2024 UTC
+# Version 2024111900, Last Updated Tue Nov 19 07:07:01 2024 UTC
AAA
AARP
ABB
diff --git a/library/cpp/yt/misc/cast-inl.h b/library/cpp/yt/misc/cast-inl.h
index 50920f0193..5e5a0d90ee 100644
--- a/library/cpp/yt/misc/cast-inl.h
+++ b/library/cpp/yt/misc/cast-inl.h
@@ -119,7 +119,7 @@ T CheckedIntegralCast(S value)
template <class T, class S>
requires TEnumTraits<T>::IsEnum
-constexpr std::optional<T> TryCheckedEnumCast(S value)
+constexpr std::optional<T> TryCheckedEnumCast(S value, bool enableUnknown)
{
auto underlying = TryCheckedIntegralCast<std::underlying_type_t<T>>(value);
[[unlikely]] if (!underlying) {
@@ -127,6 +127,15 @@ constexpr std::optional<T> TryCheckedEnumCast(S value)
}
auto candidate = static_cast<T>(*underlying);
[[unlikely]] if (!TEnumTraits<T>::IsValidValue(candidate)) {
+ if (enableUnknown) {
+ if constexpr (constexpr auto optionalUnknownValue = TEnumTraits<T>::TryGetUnknownValue()) {
+ if constexpr (TEnumTraits<T>::IsBitEnum) {
+ return static_cast<T>(*underlying & ToUnderlying(TEnumTraits<T>::GetAllSetValue())) | *optionalUnknownValue;
+ } else {
+ return *optionalUnknownValue;
+ }
+ }
+ }
return std::nullopt;
}
return candidate;
@@ -136,7 +145,7 @@ template <class T, class S>
requires TEnumTraits<T>::IsEnum
T CheckedEnumCast(S value)
{
- auto result = TryCheckedEnumCast<T>(value);
+ auto result = TryCheckedEnumCast<T>(value, /*enableUnknown*/ true);
[[unlikely]] if (!result) {
if constexpr (std::is_signed_v<S>) {
throw TSimpleException(Sprintf("Error casting %s value \"%" PRIi64 "\" to enum %s",
diff --git a/library/cpp/yt/misc/cast.h b/library/cpp/yt/misc/cast.h
index a4e32e0287..ccf1eb21a6 100644
--- a/library/cpp/yt/misc/cast.h
+++ b/library/cpp/yt/misc/cast.h
@@ -26,7 +26,7 @@ T CheckedIntegralCast(S value);
template <class T, class S>
requires TEnumTraits<T>::IsEnum
-constexpr std::optional<T> TryCheckedEnumCast(S value);
+constexpr std::optional<T> TryCheckedEnumCast(S value, bool enableUnknown = false);
template <class T, class S>
requires TEnumTraits<T>::IsEnum
diff --git a/library/cpp/yt/misc/unittests/cast_ut.cpp b/library/cpp/yt/misc/unittests/cast_ut.cpp
index 8662575d6c..e6f5409f2f 100644
--- a/library/cpp/yt/misc/unittests/cast_ut.cpp
+++ b/library/cpp/yt/misc/unittests/cast_ut.cpp
@@ -21,6 +21,26 @@ DEFINE_BIT_ENUM_WITH_UNDERLYING_TYPE(EFeatures, ui8,
((Second)(0x0002))
);
+DEFINE_BIT_ENUM(ELangsWithUnknown,
+ ((None) (0x00))
+ ((Cpp) (0x01))
+ ((Go) (0x02))
+ ((Rust) (0x04))
+ ((Python) (0x08))
+ ((JavaScript) (0x10))
+ ((CppGo) (0x03))
+ ((All) (0x1f))
+ ((Unknown) (0x20))
+);
+DEFINE_ENUM_UNKNOWN_VALUE(ELangsWithUnknown, Unknown);
+
+DEFINE_ENUM(EColorWithUnknown,
+ ((Red) (0))
+ ((Green) (1))
+ ((Unknown) (2))
+);
+DEFINE_ENUM_UNKNOWN_VALUE(EColorWithUnknown, Unknown);
+
TEST(TCastTest, TryCheckedEnumCast)
{
EXPECT_EQ((TryCheckedEnumCast<ECardinal, char>(2)), ECardinal::East);
@@ -36,6 +56,35 @@ TEST(TCastTest, TryCheckedEnumCast)
EXPECT_EQ((TryCheckedEnumCast<EFeatures, ui8>(ToUnderlying(EFeatures::First | EFeatures::Second))), EFeatures::First | EFeatures::Second);
EXPECT_FALSE((TryCheckedEnumCast<EFeatures, ui8>(0x10)));
+
+ EXPECT_FALSE(TryCheckedEnumCast<EColorWithUnknown>(3));
+ EXPECT_EQ(TryCheckedEnumCast<EColorWithUnknown>(3, /*enableUnknown*/ true), EColorWithUnknown::Unknown);
+
+ EXPECT_FALSE(TryCheckedEnumCast<ELangsWithUnknown>(0x40));
+ EXPECT_EQ(TryCheckedEnumCast<ELangsWithUnknown>(0x40, /*enableUnknown*/ true), ELangsWithUnknown::Unknown);
+ EXPECT_EQ(TryCheckedEnumCast<ELangsWithUnknown>(0x41, /*enableUnknown*/ true), ELangsWithUnknown::Unknown | ELangsWithUnknown::Cpp);
+}
+
+TEST(TCastTest, CheckedEnumCast)
+{
+ EXPECT_EQ((CheckedEnumCast<ECardinal, char>(2)), ECardinal::East);
+ EXPECT_EQ((CheckedEnumCast<ECardinal, int>(3)), ECardinal::South);
+
+ EXPECT_THROW((CheckedEnumCast<ECardinal, char>(100)), TSimpleException);
+ EXPECT_THROW((CheckedEnumCast<ECardinal, int>(300)), TSimpleException);
+
+ EXPECT_EQ((CheckedEnumCast<EFeatures, ui8>(0)), EFeatures::None);
+ EXPECT_EQ((CheckedEnumCast<EFeatures, ui8>(ToUnderlying(EFeatures::First))), EFeatures::First);
+ EXPECT_EQ((CheckedEnumCast<EFeatures, ui8>(ToUnderlying(EFeatures::Second))), EFeatures::Second);
+ EXPECT_EQ((CheckedEnumCast<EFeatures, int>(ToUnderlying(EFeatures::First))), EFeatures::First);
+ EXPECT_EQ((CheckedEnumCast<EFeatures, ui8>(ToUnderlying(EFeatures::First | EFeatures::Second))), EFeatures::First | EFeatures::Second);
+
+ EXPECT_THROW((CheckedEnumCast<EFeatures, ui8>(0x10)), TSimpleException);
+
+ EXPECT_EQ(CheckedEnumCast<EColorWithUnknown>(3), EColorWithUnknown::Unknown);
+
+ EXPECT_EQ(CheckedEnumCast<ELangsWithUnknown>(0x40), ELangsWithUnknown::Unknown);
+ EXPECT_EQ(CheckedEnumCast<ELangsWithUnknown>(0x41), ELangsWithUnknown::Unknown | ELangsWithUnknown::Cpp);
}
////////////////////////////////////////////////////////////////////////////////
diff --git a/library/cpp/yt/string/format-inl.h b/library/cpp/yt/string/format-inl.h
index 7f1f725966..e90d68bfe1 100644
--- a/library/cpp/yt/string/format-inl.h
+++ b/library/cpp/yt/string/format-inl.h
@@ -871,12 +871,43 @@ concept CFormatter = CInvocable<T, void(size_t, TStringBuilderBase*, TStringBuf)
////////////////////////////////////////////////////////////////////////////////
template <CFormatter TFormatter>
-void RunFormatter(
+void RunFormatterAt(
+ const TFormatter& formatter,
+ size_t index,
+ TStringBuilderBase* builder,
+ TStringBuf spec,
+ bool singleQuotes,
+ bool doubleQuotes)
+{
+ // 'n' means 'nothing'; skip the argument.
+ if (!spec.Contains('n')) {
+ if (singleQuotes) {
+ builder->AppendChar('\'');
+ }
+ if (doubleQuotes) {
+ builder->AppendChar('"');
+ }
+
+ formatter(index, builder, spec);
+
+ if (singleQuotes) {
+ builder->AppendChar('\'');
+ }
+ if (doubleQuotes) {
+ builder->AppendChar('"');
+ }
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <CFormatter TFormatter>
+void RunFormatterFullScan(
TStringBuilderBase* builder,
TStringBuf format,
- const TFormatter& formatter)
+ const TFormatter& formatter,
+ int argIndex = 0)
{
- size_t argIndex = 0;
auto current = std::begin(format);
auto end = std::end(format);
while (true) {
@@ -912,27 +943,13 @@ void RunFormatter(
bool singleQuotes = false;
bool doubleQuotes = false;
+ static constexpr TStringBuf conversionSpecifiers =
+ "diuoxXfFeEgGaAcspn";
+
while (
argFormatEnd != end &&
- *argFormatEnd != GenericSpecSymbol && // value in generic format
- *argFormatEnd != 'd' && // others are standard specifiers supported by printf
- *argFormatEnd != 'i' &&
- *argFormatEnd != 'u' &&
- *argFormatEnd != 'o' &&
- *argFormatEnd != 'x' &&
- *argFormatEnd != 'X' &&
- *argFormatEnd != 'f' &&
- *argFormatEnd != 'F' &&
- *argFormatEnd != 'e' &&
- *argFormatEnd != 'E' &&
- *argFormatEnd != 'g' &&
- *argFormatEnd != 'G' &&
- *argFormatEnd != 'a' &&
- *argFormatEnd != 'A' &&
- *argFormatEnd != 'c' &&
- *argFormatEnd != 's' &&
- *argFormatEnd != 'p' &&
- *argFormatEnd != 'n')
+ *argFormatEnd != GenericSpecSymbol && // value in generic format
+ !conversionSpecifiers.Contains(*argFormatEnd)) // others are standard specifiers supported by printf
{
switch (*argFormatEnd) {
case 'q':
@@ -952,27 +969,162 @@ void RunFormatter(
++argFormatEnd;
}
- // 'n' means 'nothing'; skip the argument.
- if (*argFormatBegin != 'n') {
- // Format argument.
- TStringBuf argFormat(argFormatBegin, argFormatEnd);
- if (singleQuotes) {
- builder->AppendChar('\'');
- }
- if (doubleQuotes) {
- builder->AppendChar('"');
- }
- formatter(argIndex++, builder, argFormat);
- if (singleQuotes) {
- builder->AppendChar('\'');
- }
- if (doubleQuotes) {
- builder->AppendChar('"');
+ RunFormatterAt(
+ formatter,
+ argIndex++,
+ builder,
+ TStringBuf{argFormatBegin, argFormatEnd},
+ singleQuotes,
+ doubleQuotes);
+
+ current = argFormatEnd;
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <CFormatter TFormatter, class... TArgs>
+void RunFormatter(
+ TStringBuilderBase* builder,
+ TBasicFormatString<TArgs...> formatString,
+ const TFormatter& formatter)
+{
+ auto isValidLocations = [] (const auto& t) {
+ return std::get<0>(t) != std::get<1>(t);
+ };
+ // Generally marker is simply "%v" e.g. 2 symbols.
+ // We assume it is used to insert something for roughly 5 symbols
+ // of size.
+ builder->Preallocate(std::size(formatString.Get()) + sizeof...(TArgs) * (5 - 2));
+
+ // Empty marker positions -- fallback to the normal impl.
+ if constexpr (sizeof...(TArgs) != 0) {
+ if (!isValidLocations(formatString.Markers[0])) {
+ RunFormatterFullScan(builder, formatString.Get(), formatter);
+ return;
+ }
+ } else {
+ if (formatString.Escapes[0] == -2) {
+ RunFormatterFullScan(builder, formatString.Get(), formatter);
+ return;
+ }
+ }
+
+ int escapesFound = 0;
+ int currentPos = 0;
+
+ auto beginIt = formatString.Get().begin();
+ auto size = formatString.Get().size();
+
+ const auto& [markers, escapes] = std::tie(formatString.Markers, formatString.Escapes);
+
+ auto appendVerbatim = [&] (int offsetToEnd) {
+ builder->AppendString(TStringBuf{beginIt + currentPos, beginIt + offsetToEnd});
+ };
+
+ auto processEscape = [&] () mutable {
+ // OpenMP doesn't support structured bindings :(.
+ auto escapePos = formatString.Escapes[escapesFound];
+
+ // Append everything that's present until %%.
+ appendVerbatim(escapePos);
+
+ // Append '%'.
+ builder->AppendChar('%');
+
+ // Advance position to first '%' pos + 2.
+ currentPos = escapePos + 2;
+ };
+
+ int argIndex = 0;
+
+ while(argIndex < std::ssize(markers)) {
+ auto [markerStart, markerEnd] = markers[argIndex];
+
+ if (
+ escapes[escapesFound] != -1 &&
+ escapes[escapesFound] - currentPos < markerStart - currentPos)
+ {
+ // Escape sequence is closer.
+ processEscape();
+ ++escapesFound;
+ } else {
+ // Normal marker is closer.
+
+ // Append everything that's present until marker start.
+ appendVerbatim(markerStart);
+
+ // Parsing format string.
+
+ // We skip '%' here since spec does not contain it.
+ auto spec = TStringBuf{beginIt + markerStart + 1, beginIt + markerEnd};
+ bool singleQuotes = false;
+ bool doubleQuotes = false;
+ for (auto c : spec) {
+ if (c == 'q') {
+ singleQuotes = true;
+ }
+ if (c == 'Q') {
+ doubleQuotes = true;
+ }
}
+ RunFormatterAt(
+ formatter,
+ argIndex,
+ builder,
+ spec,
+ singleQuotes,
+ doubleQuotes);
+
+ // Advance position past marker's end.
+ currentPos = markerEnd;
+ ++argIndex;
+ continue;
}
- current = argFormatEnd;
+ // Check if the number of escapes we have found has exceeded the recorded limit
+ // e.g. we have to manually scan the rest of the formatString.
+ if (escapesFound == std::ssize(escapes)) {
+ break;
+ }
}
+
+ // Process remaining escapes.
+ while (escapesFound < std::ssize(escapes)) {
+ if (escapes[escapesFound] == -1) {
+ break;
+ }
+
+ processEscape();
+ ++escapesFound;
+ }
+
+ // We either ran out of markers or reached the limit of allowed
+ // escape sequences.
+
+ // Happy path: no limit on escape sequences.
+ if (escapesFound != std::ssize(escapes)) {
+ // Append whatever's left until the end.
+ appendVerbatim(size);
+ return;
+ }
+
+ // Sad path: we have to fully parse remainder of format.
+ RunFormatterFullScan(builder, TStringBuf{beginIt + currentPos, beginIt + size}, formatter, argIndex);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+// For benchmarking purposes.
+template <class... TArgs>
+TString FormatOld(TFormatString<TArgs...> format, TArgs&&... args)
+{
+ TStringBuilder builder;
+ if constexpr ((CFormattable<TArgs> && ...)) {
+ NYT::NDetail::TValueFormatter<0, TArgs...> formatter(args...);
+ NYT::NDetail::RunFormatterFullScan(&builder, format.Get(), formatter);
+ }
+ return builder.Flush();
}
} // namespace NDetail
@@ -991,7 +1143,7 @@ void Format(TStringBuilderBase* builder, TFormatString<TArgs...> format, TArgs&&
// a second error.
if constexpr ((CFormattable<TArgs> && ...)) {
NYT::NDetail::TValueFormatter<0, TArgs...> formatter(args...);
- NYT::NDetail::RunFormatter(builder, format.Get(), formatter);
+ NYT::NDetail::RunFormatter(builder, format, formatter);
}
}
@@ -1012,7 +1164,7 @@ void FormatVector(
const TVector& vec)
{
NYT::NDetail::TRangeFormatter<typename TVector::value_type> formatter(vec);
- NYT::NDetail::RunFormatter(builder, format, formatter);
+ NYT::NDetail::RunFormatterFullScan(builder, format, formatter);
}
template <class TVector>
@@ -1022,7 +1174,7 @@ void FormatVector(
const TVector& vec)
{
NYT::NDetail::TRangeFormatter<typename TVector::value_type> formatter(vec);
- NYT::NDetail::RunFormatter(builder, format, formatter);
+ NYT::NDetail::RunFormatterFullScan(builder, format, formatter);
}
template <size_t Length, class TVector>
diff --git a/library/cpp/yt/string/format_analyser.h b/library/cpp/yt/string/format_analyser.h
index 20eee60580..9f194144dc 100644
--- a/library/cpp/yt/string/format_analyser.h
+++ b/library/cpp/yt/string/format_analyser.h
@@ -4,6 +4,7 @@
#include <util/generic/strbuf.h>
+#include <algorithm>
#include <array>
#include <string_view>
@@ -14,13 +15,26 @@ namespace NYT::NDetail {
struct TFormatAnalyser
{
public:
+ using TMarkerLocation = std::tuple<int, int>;
+ // NB(arkady-e1ppa): Location is considered invalid (e.g. not filled)
+ // if get<0> == get<1> == 0.
+ template <class... TArgs>
+ using TMarkerLocations = std::array<TMarkerLocation, sizeof...(TArgs)>;
+ // NB(arkady-e1ppa): We can't cover all of them since that would require
+ // dynamic storage for their coordinates and we do not have
+ // constexpr context large enough to deallocate dynamic memory at the
+ // correct time. Thus we store first 5 position and scanning afterwards
+ // is pessimized. |-1| is for no position at all.
+ // |-2| is used to imply runtime format.
+ using TEscapeLocations = std::array<int, 5>;
+
// TODO(arkady-e1ppa): Until clang-19 consteval functions
// defined out of line produce symbols in rare cases
// causing linker to crash.
template <class... TArgs>
- static consteval void ValidateFormat(std::string_view fmt)
+ static consteval auto AnalyzeFormat(std::string_view fmt)
{
- DoValidateFormat<TArgs...>(fmt);
+ return DoAnalyzeFormat<TArgs...>(fmt);
}
private:
@@ -51,11 +65,16 @@ private:
static constexpr char IntroductorySymbol = '%';
template <class... TArgs>
- static consteval void DoValidateFormat(std::string_view format)
+ static consteval auto DoAnalyzeFormat(std::string_view format)
{
- std::array<std::string_view, sizeof...(TArgs)> markers = {};
std::array<TSpecifiers, sizeof...(TArgs)> specifiers{GetSpecifiers<TArgs>()...};
+ TMarkerLocations<TArgs...> markerLocations = {};
+ TEscapeLocations escapeLocations = {};
+ std::ranges::fill(escapeLocations, -1);
+
+ int escapesCount = 0;
+
int markerCount = 0;
int currentMarkerStart = -1;
@@ -81,12 +100,17 @@ private:
throw "You may not terminate flag sequence other than %% with \'%\' symbol";
}
// '%%' detected --- skip
+ if (escapesCount < std::ssize(escapeLocations)) {
+ escapeLocations[escapesCount] = currentMarkerStart;
+ ++escapesCount;
+ }
+
currentMarkerStart = -1;
continue;
}
// We are inside of marker.
- if (markerCount == std::ssize(markers)) {
+ if (markerCount == std::ssize(markerLocations)) {
// Too many markers
throw "Number of arguments supplied to format is smaller than the number of flag sequences";
}
@@ -94,8 +118,8 @@ private:
if (Contains(specifiers[markerCount].Conversion, symbol)) {
// Marker has finished.
- markers[markerCount]
- = std::string_view(format.begin() + currentMarkerStart, index - currentMarkerStart + 1);
+ markerLocations[markerCount]
+ = std::tuple{currentMarkerStart, index + 1};
currentMarkerStart = -1;
++markerCount;
@@ -110,16 +134,16 @@ private:
if (currentMarkerStart != -1) {
// Runaway marker.
throw "Unterminated flag sequence detected; Use \'%%\' to type plain %";
- return;
}
- if (markerCount < std::ssize(markers)) {
+ if (markerCount < std::ssize(markerLocations)) {
// Missing markers.
throw "Number of arguments supplied to format is greater than the number of flag sequences";
}
// TODO(arkady-e1ppa): Consider per-type verification
// of markers.
+ return std::tuple(markerLocations, escapeLocations);
}
};
diff --git a/library/cpp/yt/string/format_arg.h b/library/cpp/yt/string/format_arg.h
index 544e265766..4dc7be06e8 100644
--- a/library/cpp/yt/string/format_arg.h
+++ b/library/cpp/yt/string/format_arg.h
@@ -32,7 +32,7 @@ class TFormatArgBase
public:
// TODO(arkady-e1ppa): Consider more strict formatting rules.
static constexpr std::array ConversionSpecifiers = {
- 'v', '1', 'c', 's', 'd', 'i', 'o',
+ 'v', 'c', 's', 'd', 'i', 'o',
'x', 'X', 'u', 'f', 'F', 'e', 'E',
'a', 'A', 'g', 'G', 'n', 'p'
};
diff --git a/library/cpp/yt/string/format_string-inl.h b/library/cpp/yt/string/format_string-inl.h
index a692d9648d..67f9bad45c 100644
--- a/library/cpp/yt/string/format_string-inl.h
+++ b/library/cpp/yt/string/format_string-inl.h
@@ -4,6 +4,8 @@
#include "format_string.h"
#endif
+#include <algorithm>
+
namespace NYT {
////////////////////////////////////////////////////////////////////////////////
@@ -15,8 +17,13 @@ consteval TBasicFormatString<TArgs...>::TBasicFormatString(const T& fmt)
: Format_(fmt)
{
CheckFormattability();
-#if !defined(NDEBUG) && !defined(YT_DISABLE_FORMAT_STATIC_ANALYSIS)
- NDetail::TFormatAnalyser::ValidateFormat<std::remove_cvref_t<TArgs>...>(Format_);
+#if !defined(YT_DISABLE_FORMAT_STATIC_ANALYSIS)
+ std::tie(Markers, Escapes) = NDetail::TFormatAnalyser::AnalyzeFormat<std::remove_cvref_t<TArgs>...>(Format_);
+#else
+ std::ranges::fill_n(std::ranges::begin(Escapes), 1, -1);
+ if constexpr (sizeof...(TArgs) != 0) {
+ std::ranges::fill_n(std::ranges::begin(Markers), 1, std::tuple{0, 0});
+ }
#endif
}
@@ -46,6 +53,11 @@ template <class... TArgs>
TBasicFormatString<TArgs...>::TBasicFormatString(TRuntimeFormat fmt)
: Format_(fmt.Get())
{
+ std::ranges::fill_n(std::ranges::begin(Escapes), 1, -1);
+ if constexpr (sizeof...(TArgs) != 0) {
+ std::ranges::fill_n(std::ranges::begin(Markers), 1, std::tuple{0, 0});
+ }
+
// NB(arkady-e1ppa): StaticFormat performs the
// formattability check of the args in a way
// that provides more useful information
diff --git a/library/cpp/yt/string/format_string.h b/library/cpp/yt/string/format_string.h
index 786c2e39ed..1008ccb453 100644
--- a/library/cpp/yt/string/format_string.h
+++ b/library/cpp/yt/string/format_string.h
@@ -43,6 +43,10 @@ public:
static consteval void CheckFormattability();
+ // Data used for compile-time slicing of the format string.
+ NDetail::TFormatAnalyser::TMarkerLocations<TArgs...> Markers = {};
+ NDetail::TFormatAnalyser::TEscapeLocations Escapes = {};
+
private:
std::string_view Format_;
diff --git a/library/cpp/yt/string/unittests/format_ut.cpp b/library/cpp/yt/string/unittests/format_ut.cpp
index ac3be99ad9..8aca8c8e29 100644
--- a/library/cpp/yt/string/unittests/format_ut.cpp
+++ b/library/cpp/yt/string/unittests/format_ut.cpp
@@ -326,6 +326,11 @@ TEST(TFormatTest, CustomFlagsCollectionTwoLevels)
EXPECT_EQ(Format("%NRv", arr), toCollectionD2("RNP"));
}
+TEST(TFormatTest, ManyEscapes)
+{
+ EXPECT_EQ("a%b%c%d%e%f%g", Format("%v%%%v%%%v%%%v%%%v%%%v%%%g", "a", "b", "c", "d", "e", "f", "g"));
+}
+
////////////////////////////////////////////////////////////////////////////////
} // namespace
diff --git a/library/python/testing/yatest_common/yatest/common/runtime.py b/library/python/testing/yatest_common/yatest/common/runtime.py
index 9d2fe5932a..e4f246d5c8 100644
--- a/library/python/testing/yatest_common/yatest/common/runtime.py
+++ b/library/python/testing/yatest_common/yatest/common/runtime.py
@@ -190,6 +190,8 @@ def data_path(path=None):
:param path: path relative to the arcadia_tests_data directory, e.g. yatest.common.data_path("pers/rerank_service")
:return: absolute path inside arcadia_tests_data
"""
+ if "USE_ATD_FROM_SNAPSHOT" in os.environ:
+ return os.path.join(source_path(), "atd_ro_snapshot", path)
return _join_path(_get_ya_plugin_instance().data_root, path)