diff options
author | Maxim Yurchuk <maxim-yurchuk@ydb.tech> | 2024-11-20 17:37:57 +0000 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-11-20 17:37:57 +0000 |
commit | f76323e9b295c15751e51e3443aa47a36bee8023 (patch) | |
tree | 4113c8cad473a33e0f746966e0cf087252fa1d7a /library | |
parent | 753ecb8d410a4cb459c26f3a0082fb2d1724fe63 (diff) | |
parent | a7b9a6afea2a9d7a7bfac4c5eb4c1a8e60adb9e6 (diff) | |
download | ydb-f76323e9b295c15751e51e3443aa47a36bee8023.tar.gz |
Merge pull request #11788 from ydb-platform/mergelibs-241120-1113
Library import 241120-1113
Diffstat (limited to 'library')
25 files changed, 336 insertions, 440 deletions
diff --git a/library/cpp/lwtrace/rwspinlock.h b/library/cpp/lwtrace/rwspinlock.h index 5e4608e068..1effd5fe0e 100644 --- a/library/cpp/lwtrace/rwspinlock.h +++ b/library/cpp/lwtrace/rwspinlock.h @@ -1,6 +1,6 @@ #pragma once -#include <library/cpp/deprecated/atomic/atomic.h> +#include <atomic> #include <util/system/spinlock.h> @@ -27,16 +27,16 @@ // * writer can release lock (State = 0: -> READING) struct TRWSpinLock { - TAtomic State; // must be initialized by 'TRWSpinLock myLock = {0};' construction + std::atomic_signed_lock_free State; void Init() noexcept { - State = 0; + State.store(0, std::memory_order_relaxed); } void AcquireRead() noexcept { while (true) { - TAtomic a = AtomicGet(State); - if ((a & 1) == 0 && AtomicCas(&State, a + 2, a)) { + std::atomic_signed_lock_free::value_type a = State.load(std::memory_order_acquire); + if ((a & 1) == 0 && State.compare_exchange_strong(a, a + 2, std::memory_order_acquire)) { break; } SpinLockPause(); @@ -44,25 +44,29 @@ struct TRWSpinLock { } void ReleaseRead() noexcept { - AtomicAdd(State, -2); + State.fetch_add(-2, std::memory_order_release); } void AcquireWrite() noexcept { while (true) { - TAtomic a = AtomicGet(State); - if ((a & 1) == 0 && AtomicCas(&State, a + 1, a)) { + std::atomic_signed_lock_free::value_type a = State.load(std::memory_order_acquire); + if ((a & 1) == 0 && State.compare_exchange_strong(a, a + 1, std::memory_order_acquire)) { break; } SpinLockPause(); } - while (!AtomicCas(&State, TAtomicBase(-1), 1)) { + while (true) { + std::atomic_signed_lock_free::value_type a = 1; + if (State.compare_exchange_strong(a, -1, std::memory_order_acquire)) { + break; + } SpinLockPause(); } } void ReleaseWrite() noexcept { - AtomicSet(State, 0); + State.store(0, std::memory_order_release); } }; diff --git a/library/cpp/netliba/v6/udp_address.cpp b/library/cpp/netliba/v6/udp_address.cpp index ba5fd6f45b..be823e2dfa 100644 --- a/library/cpp/netliba/v6/udp_address.cpp +++ b/library/cpp/netliba/v6/udp_address.cpp @@ -196,7 +196,7 @@ namespace NNetliba { continue; } if (ptr->Mtu < 1280) { - fprintf(stderr, "WARNING: MTU %d is less then ipv6 minimum", ptr->Mtu); + fprintf(stderr, "WARNING: MTU %lu is less then ipv6 minimum", ptr->Mtu); } for (IP_ADAPTER_UNICAST_ADDRESS* addr = ptr->FirstUnicastAddress; addr; addr = addr->Next) { sockaddr* x = (sockaddr*)addr->Address.lpSockaddr; diff --git a/library/cpp/pop_count/benchmark/main.cpp b/library/cpp/pop_count/benchmark/main.cpp deleted file mode 100644 index 1af2d157b6..0000000000 --- a/library/cpp/pop_count/benchmark/main.cpp +++ /dev/null @@ -1,82 +0,0 @@ -#include <util/stream/output.h> -#include <util/datetime/cputimer.h> -#include <util/system/type_name.h> - -#include <library/cpp/pop_count/popcount.h> -#include <library/cpp/testing/benchmark/bench.h> - -#include <bit> - -template <class F, class I> -inline void DoRun(F&& f, I&& i) { - const ui64 n = i.Iterations(); - - for (ui64 j = 0; j < n; ++j) { - Y_DO_NOT_OPTIMIZE_AWAY(f(j * (ui64)123456 + (ui64)1)); - } -} - -Y_CPU_BENCHMARK(PopCount_8, iface) { - DoRun([](ui8 x) { - return PopCount<ui8>(x); - }, - iface); -} - -Y_CPU_BENCHMARK(std_popcount_8, iface) { - DoRun([](ui8 x) { - return std::popcount<ui8>(x); - }, - iface); -} - -Y_CPU_BENCHMARK(PopCount_16, iface) { - DoRun([](ui16 x) { - return PopCount<ui16>(x); - }, - iface); -} - -Y_CPU_BENCHMARK(std_popcount_16, iface) { - DoRun([](ui16 x) { - return std::popcount<ui16>(x); - }, - iface); -} - -Y_CPU_BENCHMARK(PopCount_32, iface) { - DoRun([](ui32 x) { - return PopCount<ui32>(x); - }, - iface); -} - -Y_CPU_BENCHMARK(std_popcount_32, iface) { - DoRun([](ui32 x) { - return std::popcount<ui32>(x); - }, - iface); -} - -Y_CPU_BENCHMARK(PopCount_64, iface) { - DoRun([](ui64 x) { - return PopCount<ui64>(x); - }, - iface); -} - -Y_CPU_BENCHMARK(std_popcount_64, iface) { - DoRun([](ui64 x) { - return std::popcount<ui64>(x); - }, - iface); -} - -#if !defined(_MSC_VER) -Y_CPU_BENCHMARK(BUILTIN_64, iface) { - DoRun([](ui64 x) { - return __builtin_popcountll(x); - }, - iface); -} -#endif diff --git a/library/cpp/pop_count/benchmark/ya.make b/library/cpp/pop_count/benchmark/ya.make deleted file mode 100644 index e7ea9418db..0000000000 --- a/library/cpp/pop_count/benchmark/ya.make +++ /dev/null @@ -1,12 +0,0 @@ -Y_BENCHMARK() - -PEERDIR( - util/draft - library/cpp/pop_count -) - -SRCS( - main.cpp -) - -END() diff --git a/library/cpp/pop_count/popcount.cpp b/library/cpp/pop_count/popcount.cpp deleted file mode 100644 index 49276424be..0000000000 --- a/library/cpp/pop_count/popcount.cpp +++ /dev/null @@ -1,30 +0,0 @@ -#include "popcount.h" - -#include <util/system/defaults.h> -#include <util/system/yassert.h> - -#include <string.h> - -static const ui8 PopCountLUT8Impl[1 << 8] = { -#define B2(n) n, n + 1, n + 1, n + 2 -#define B4(n) B2(n), B2(n + 1), B2(n + 1), B2(n + 2) -#define B6(n) B4(n), B4(n + 1), B4(n + 1), B4(n + 2) - B6(0), B6(1), B6(1), B6(2)}; - -ui8 const* PopCountLUT8 = PopCountLUT8Impl; - -#if !defined(_MSC_VER) -//ICE here for msvc - -static const ui8 PopCountLUT16Impl[1 << 16] = { -#define B2(n) n, n + 1, n + 1, n + 2 -#define B4(n) B2(n), B2(n + 1), B2(n + 1), B2(n + 2) -#define B6(n) B4(n), B4(n + 1), B4(n + 1), B4(n + 2) -#define B8(n) B6(n), B6(n + 1), B6(n + 1), B6(n + 2) -#define B10(n) B8(n), B8(n + 1), B8(n + 1), B8(n + 2) -#define B12(n) B10(n), B10(n + 1), B10(n + 1), B10(n + 2) -#define B14(n) B12(n), B12(n + 1), B12(n + 1), B12(n + 2) - B14(0), B14(1), B14(1), B14(2)}; - -ui8 const* PopCountLUT16 = PopCountLUT16Impl; -#endif diff --git a/library/cpp/pop_count/popcount.h b/library/cpp/pop_count/popcount.h deleted file mode 100644 index 3d67737ed2..0000000000 --- a/library/cpp/pop_count/popcount.h +++ /dev/null @@ -1,105 +0,0 @@ -#pragma once - -#include <util/generic/typelist.h> -#include <util/system/cpu_id.h> -#include <util/system/defaults.h> -#include <util/system/hi_lo.h> -#include <util/system/platform.h> - -#if defined(_MSC_VER) -#include <intrin.h> -#endif - -static inline ui32 PopCountImpl(ui8 n) { -#if defined(_ppc64_) - ui32 r; - __asm__("popcntb %0, %1" - : "=r"(r) - : "r"(n) - :); - return r; -#else - extern ui8 const* PopCountLUT8; - return PopCountLUT8[n]; -#endif -} - -static inline ui32 PopCountImpl(ui16 n) { -#if defined(_MSC_VER) - return __popcnt16(n); -#else - extern ui8 const* PopCountLUT16; - return PopCountLUT16[n]; -#endif -} - -static inline ui32 PopCountImpl(ui32 n) { -#if defined(_MSC_VER) - return __popcnt(n); -#else -#if defined(_x86_64_) - if (NX86::CachedHavePOPCNT()) { - ui32 r; - - __asm__("popcnt %1, %0;" - : "=r"(r) - : "r"(n) - :); - - return r; - } -#else -#if defined(_ppc64_) - ui32 r; - - __asm__("popcntw %0, %1" - : "=r"(r) - : "r"(n) - :); - - return r; -#endif -#endif - - return PopCountImpl((ui16)Lo16(n)) + PopCountImpl((ui16)Hi16(n)); -#endif -} - -static inline ui32 PopCountImpl(ui64 n) { -#if defined(_MSC_VER) && !defined(_i386_) - return __popcnt64(n); -#else -#if defined(_x86_64_) - if (NX86::CachedHavePOPCNT()) { - ui64 r; - - __asm__("popcnt %1, %0;" - : "=r"(r) - : "r"(n) - :); - - return r; - } -#else -#if defined(_ppc64_) - ui32 r; - - __asm__("popcntd %0, %1" - : "=r"(r) - : "r"(n) - :); - - return r; -#endif -#endif - - return PopCountImpl((ui32)Lo32(n)) + PopCountImpl((ui32)Hi32(n)); -#endif -} - -template <class T> -static inline ui32 PopCount(T n) { - using TCvt = TFixedWidthUnsignedInt<T>; - - return PopCountImpl((TCvt)n); -} diff --git a/library/cpp/pop_count/popcount_ut.cpp b/library/cpp/pop_count/popcount_ut.cpp deleted file mode 100644 index 5cd6605411..0000000000 --- a/library/cpp/pop_count/popcount_ut.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include "popcount.h" - -#include <library/cpp/testing/unittest/registar.h> - -#include <util/random/random.h> - -Y_UNIT_TEST_SUITE(TestPopCount) { - template <class T> - static inline ui32 SlowPopCount(T t) { - ui32 ret = 0; - - while (t) { - if (t & T(1)) { - ++ret; - } - - t = t >> 1; - } - - return ret; - } - - template <class T> - static inline void Test() { - for (size_t i = 0; i < 10000; ++i) { - const T rndv = RandomNumber<T>(); - - UNIT_ASSERT_VALUES_EQUAL(SlowPopCount(rndv), PopCount(rndv)); - } - } - - Y_UNIT_TEST(Test8) { - Test<ui8>(); - } - - Y_UNIT_TEST(Test16) { - Test<ui16>(); - } - - Y_UNIT_TEST(Test32) { - Test<ui32>(); - } - - Y_UNIT_TEST(Test64) { - Test<ui64>(); - } - - Y_UNIT_TEST(TestPopCount) { - UNIT_ASSERT_VALUES_EQUAL(PopCount(0), 0); - UNIT_ASSERT_VALUES_EQUAL(PopCount(1), 1); - UNIT_ASSERT_VALUES_EQUAL(PopCount(1 << 10), 1); - UNIT_ASSERT_VALUES_EQUAL(PopCount((1 << 10) + 1), 2); - UNIT_ASSERT_VALUES_EQUAL(PopCount(0xFFFF), 16); - UNIT_ASSERT_VALUES_EQUAL(PopCount(0xFFFFFFFF), 32); - UNIT_ASSERT_VALUES_EQUAL(PopCount(0x55555555), 16); - - UNIT_ASSERT_VALUES_EQUAL(0, PopCount(0ULL)); - UNIT_ASSERT_VALUES_EQUAL(1, PopCount(1ULL)); - UNIT_ASSERT_VALUES_EQUAL(16, PopCount(0xAAAAAAAAULL)); - UNIT_ASSERT_VALUES_EQUAL(32, PopCount(0xFFFFFFFFULL)); - UNIT_ASSERT_VALUES_EQUAL(32, PopCount(0xAAAAAAAAAAAAAAAAULL)); - UNIT_ASSERT_VALUES_EQUAL(64, PopCount(0xFFFFFFFFFFFFFFFFULL)); - - ui64 v = 0; - - for (int i = 0; i < 64; v |= 1ULL << i, ++i) { - UNIT_ASSERT_VALUES_EQUAL(i, PopCount(v)); - } - } -} diff --git a/library/cpp/pop_count/ut/ya.make b/library/cpp/pop_count/ut/ya.make deleted file mode 100644 index 179d47f064..0000000000 --- a/library/cpp/pop_count/ut/ya.make +++ /dev/null @@ -1,7 +0,0 @@ -UNITTEST_FOR(library/cpp/pop_count) - -SRCS( - popcount_ut.cpp -) - -END() diff --git a/library/cpp/pop_count/ya.make b/library/cpp/pop_count/ya.make deleted file mode 100644 index d858f3e87e..0000000000 --- a/library/cpp/pop_count/ya.make +++ /dev/null @@ -1,12 +0,0 @@ -LIBRARY() - -SRCS( - popcount.cpp -) - -END() - -RECURSE( - benchmark - ut -) diff --git a/library/cpp/string_utils/ztstrbuf/ztstrbuf_ut.cpp b/library/cpp/string_utils/ztstrbuf/ztstrbuf_ut.cpp index d8215abdfa..b22d4bf216 100644 --- a/library/cpp/string_utils/ztstrbuf/ztstrbuf_ut.cpp +++ b/library/cpp/string_utils/ztstrbuf/ztstrbuf_ut.cpp @@ -12,7 +12,7 @@ Y_UNIT_TEST_SUITE(TZtStringBufTest) { Y_UNIT_TEST(Constness) { constexpr TZtStringBuf s0{"bar"}; static_assert(s0[0] == 'b'); - static_assert(s0[3] == '\0'); + static_assert(s0.data()[s0.size()] == '\0'); static_assert(s0.data()[2] == 'r'); UNIT_ASSERT_VALUES_EQUAL(s0, TString{"bar"}); } diff --git a/library/cpp/svnversion/ya.make b/library/cpp/svnversion/ya.make index d028f45c94..27e9154644 100644 --- a/library/cpp/svnversion/ya.make +++ b/library/cpp/svnversion/ya.make @@ -6,7 +6,7 @@ SRCS( ) IF (OPENSOURCE_PROJECT == "yt-cpp-sdk") - PEERDIR(build/scripts/c_templates/) + PEERDIR(build/scripts/c_templates) ENDIF() END() diff --git a/library/cpp/testing/common/env.cpp b/library/cpp/testing/common/env.cpp index 13f3f186af..67e081c371 100644 --- a/library/cpp/testing/common/env.cpp +++ b/library/cpp/testing/common/env.cpp @@ -41,6 +41,10 @@ TString BinaryPath(TStringBuf path) { } TString GetArcadiaTestsData() { + if (GetEnv("USE_ATD_FROM_SNAPSHOT")) { + return ArcadiaSourceRoot() + "/atd_ro_snapshot"; + } + TString atdRoot = NPrivate::GetTestEnv().ArcadiaTestsDataDir; if (atdRoot) { return atdRoot; diff --git a/library/cpp/testing/mock_server/ut/server_ut.cpp b/library/cpp/testing/mock_server/ut/server_ut.cpp deleted file mode 100644 index 74b3dcec05..0000000000 --- a/library/cpp/testing/mock_server/ut/server_ut.cpp +++ /dev/null @@ -1,40 +0,0 @@ -#include <library/cpp/testing/mock_server/server.h> - -#include <library/cpp/http/simple/http_client.h> -#include <library/cpp/testing/unittest/registar.h> - -Y_UNIT_TEST_SUITE(Server) { - int i; - - Y_UNIT_TEST(pong) { - TPortManager pm; - ui16 port = pm.GetPort(80); - NMock::TMockServer server(port, []() { return new NMock::TPong; }); - - TKeepAliveHttpClient cl("localhost", port); - UNIT_ASSERT_VALUES_EQUAL(200, cl.DoGet("/ping")); - UNIT_ASSERT_VALUES_EQUAL(404, cl.DoGet("/kek")); - } - - Y_UNIT_TEST(custom) { - class TCustomReplier: public TRequestReplier { - public: - bool DoReply(const TReplyParams& params) override { - THttpResponse resp(HttpCodes::HTTP_OK); - resp.SetContent("everithing is ok"); - resp.OutTo(params.Output); - - return true; - } - }; - - TPortManager pm; - ui16 port = pm.GetPort(80); - NMock::TMockServer server(port, []() { return new TCustomReplier; }); - - TKeepAliveHttpClient cl("localhost", port); - TStringStream out; - UNIT_ASSERT_VALUES_EQUAL(200, cl.DoGet("/foo", &out)); - UNIT_ASSERT_VALUES_EQUAL("everithing is ok", out.Str()); - } -} diff --git a/library/cpp/testing/mock_server/ut/ya.make b/library/cpp/testing/mock_server/ut/ya.make deleted file mode 100644 index f4ee34458f..0000000000 --- a/library/cpp/testing/mock_server/ut/ya.make +++ /dev/null @@ -1,11 +0,0 @@ -UNITTEST_FOR(library/cpp/testing/mock_server) - -PEERDIR( - library/cpp/http/simple -) - -SRCS( - server_ut.cpp -) - -END() diff --git a/library/cpp/tld/tlds-alpha-by-domain.txt b/library/cpp/tld/tlds-alpha-by-domain.txt index 75edec6f55..dc6477cc73 100644 --- a/library/cpp/tld/tlds-alpha-by-domain.txt +++ b/library/cpp/tld/tlds-alpha-by-domain.txt @@ -1,4 +1,4 @@ -# Version 2024111300, Last Updated Wed Nov 13 07:07:01 2024 UTC +# Version 2024111900, Last Updated Tue Nov 19 07:07:01 2024 UTC AAA AARP ABB diff --git a/library/cpp/yt/misc/cast-inl.h b/library/cpp/yt/misc/cast-inl.h index 50920f0193..5e5a0d90ee 100644 --- a/library/cpp/yt/misc/cast-inl.h +++ b/library/cpp/yt/misc/cast-inl.h @@ -119,7 +119,7 @@ T CheckedIntegralCast(S value) template <class T, class S> requires TEnumTraits<T>::IsEnum -constexpr std::optional<T> TryCheckedEnumCast(S value) +constexpr std::optional<T> TryCheckedEnumCast(S value, bool enableUnknown) { auto underlying = TryCheckedIntegralCast<std::underlying_type_t<T>>(value); [[unlikely]] if (!underlying) { @@ -127,6 +127,15 @@ constexpr std::optional<T> TryCheckedEnumCast(S value) } auto candidate = static_cast<T>(*underlying); [[unlikely]] if (!TEnumTraits<T>::IsValidValue(candidate)) { + if (enableUnknown) { + if constexpr (constexpr auto optionalUnknownValue = TEnumTraits<T>::TryGetUnknownValue()) { + if constexpr (TEnumTraits<T>::IsBitEnum) { + return static_cast<T>(*underlying & ToUnderlying(TEnumTraits<T>::GetAllSetValue())) | *optionalUnknownValue; + } else { + return *optionalUnknownValue; + } + } + } return std::nullopt; } return candidate; @@ -136,7 +145,7 @@ template <class T, class S> requires TEnumTraits<T>::IsEnum T CheckedEnumCast(S value) { - auto result = TryCheckedEnumCast<T>(value); + auto result = TryCheckedEnumCast<T>(value, /*enableUnknown*/ true); [[unlikely]] if (!result) { if constexpr (std::is_signed_v<S>) { throw TSimpleException(Sprintf("Error casting %s value \"%" PRIi64 "\" to enum %s", diff --git a/library/cpp/yt/misc/cast.h b/library/cpp/yt/misc/cast.h index a4e32e0287..ccf1eb21a6 100644 --- a/library/cpp/yt/misc/cast.h +++ b/library/cpp/yt/misc/cast.h @@ -26,7 +26,7 @@ T CheckedIntegralCast(S value); template <class T, class S> requires TEnumTraits<T>::IsEnum -constexpr std::optional<T> TryCheckedEnumCast(S value); +constexpr std::optional<T> TryCheckedEnumCast(S value, bool enableUnknown = false); template <class T, class S> requires TEnumTraits<T>::IsEnum diff --git a/library/cpp/yt/misc/unittests/cast_ut.cpp b/library/cpp/yt/misc/unittests/cast_ut.cpp index 8662575d6c..e6f5409f2f 100644 --- a/library/cpp/yt/misc/unittests/cast_ut.cpp +++ b/library/cpp/yt/misc/unittests/cast_ut.cpp @@ -21,6 +21,26 @@ DEFINE_BIT_ENUM_WITH_UNDERLYING_TYPE(EFeatures, ui8, ((Second)(0x0002)) ); +DEFINE_BIT_ENUM(ELangsWithUnknown, + ((None) (0x00)) + ((Cpp) (0x01)) + ((Go) (0x02)) + ((Rust) (0x04)) + ((Python) (0x08)) + ((JavaScript) (0x10)) + ((CppGo) (0x03)) + ((All) (0x1f)) + ((Unknown) (0x20)) +); +DEFINE_ENUM_UNKNOWN_VALUE(ELangsWithUnknown, Unknown); + +DEFINE_ENUM(EColorWithUnknown, + ((Red) (0)) + ((Green) (1)) + ((Unknown) (2)) +); +DEFINE_ENUM_UNKNOWN_VALUE(EColorWithUnknown, Unknown); + TEST(TCastTest, TryCheckedEnumCast) { EXPECT_EQ((TryCheckedEnumCast<ECardinal, char>(2)), ECardinal::East); @@ -36,6 +56,35 @@ TEST(TCastTest, TryCheckedEnumCast) EXPECT_EQ((TryCheckedEnumCast<EFeatures, ui8>(ToUnderlying(EFeatures::First | EFeatures::Second))), EFeatures::First | EFeatures::Second); EXPECT_FALSE((TryCheckedEnumCast<EFeatures, ui8>(0x10))); + + EXPECT_FALSE(TryCheckedEnumCast<EColorWithUnknown>(3)); + EXPECT_EQ(TryCheckedEnumCast<EColorWithUnknown>(3, /*enableUnknown*/ true), EColorWithUnknown::Unknown); + + EXPECT_FALSE(TryCheckedEnumCast<ELangsWithUnknown>(0x40)); + EXPECT_EQ(TryCheckedEnumCast<ELangsWithUnknown>(0x40, /*enableUnknown*/ true), ELangsWithUnknown::Unknown); + EXPECT_EQ(TryCheckedEnumCast<ELangsWithUnknown>(0x41, /*enableUnknown*/ true), ELangsWithUnknown::Unknown | ELangsWithUnknown::Cpp); +} + +TEST(TCastTest, CheckedEnumCast) +{ + EXPECT_EQ((CheckedEnumCast<ECardinal, char>(2)), ECardinal::East); + EXPECT_EQ((CheckedEnumCast<ECardinal, int>(3)), ECardinal::South); + + EXPECT_THROW((CheckedEnumCast<ECardinal, char>(100)), TSimpleException); + EXPECT_THROW((CheckedEnumCast<ECardinal, int>(300)), TSimpleException); + + EXPECT_EQ((CheckedEnumCast<EFeatures, ui8>(0)), EFeatures::None); + EXPECT_EQ((CheckedEnumCast<EFeatures, ui8>(ToUnderlying(EFeatures::First))), EFeatures::First); + EXPECT_EQ((CheckedEnumCast<EFeatures, ui8>(ToUnderlying(EFeatures::Second))), EFeatures::Second); + EXPECT_EQ((CheckedEnumCast<EFeatures, int>(ToUnderlying(EFeatures::First))), EFeatures::First); + EXPECT_EQ((CheckedEnumCast<EFeatures, ui8>(ToUnderlying(EFeatures::First | EFeatures::Second))), EFeatures::First | EFeatures::Second); + + EXPECT_THROW((CheckedEnumCast<EFeatures, ui8>(0x10)), TSimpleException); + + EXPECT_EQ(CheckedEnumCast<EColorWithUnknown>(3), EColorWithUnknown::Unknown); + + EXPECT_EQ(CheckedEnumCast<ELangsWithUnknown>(0x40), ELangsWithUnknown::Unknown); + EXPECT_EQ(CheckedEnumCast<ELangsWithUnknown>(0x41), ELangsWithUnknown::Unknown | ELangsWithUnknown::Cpp); } //////////////////////////////////////////////////////////////////////////////// diff --git a/library/cpp/yt/string/format-inl.h b/library/cpp/yt/string/format-inl.h index 7f1f725966..e90d68bfe1 100644 --- a/library/cpp/yt/string/format-inl.h +++ b/library/cpp/yt/string/format-inl.h @@ -871,12 +871,43 @@ concept CFormatter = CInvocable<T, void(size_t, TStringBuilderBase*, TStringBuf) //////////////////////////////////////////////////////////////////////////////// template <CFormatter TFormatter> -void RunFormatter( +void RunFormatterAt( + const TFormatter& formatter, + size_t index, + TStringBuilderBase* builder, + TStringBuf spec, + bool singleQuotes, + bool doubleQuotes) +{ + // 'n' means 'nothing'; skip the argument. + if (!spec.Contains('n')) { + if (singleQuotes) { + builder->AppendChar('\''); + } + if (doubleQuotes) { + builder->AppendChar('"'); + } + + formatter(index, builder, spec); + + if (singleQuotes) { + builder->AppendChar('\''); + } + if (doubleQuotes) { + builder->AppendChar('"'); + } + } +} + +//////////////////////////////////////////////////////////////////////////////// + +template <CFormatter TFormatter> +void RunFormatterFullScan( TStringBuilderBase* builder, TStringBuf format, - const TFormatter& formatter) + const TFormatter& formatter, + int argIndex = 0) { - size_t argIndex = 0; auto current = std::begin(format); auto end = std::end(format); while (true) { @@ -912,27 +943,13 @@ void RunFormatter( bool singleQuotes = false; bool doubleQuotes = false; + static constexpr TStringBuf conversionSpecifiers = + "diuoxXfFeEgGaAcspn"; + while ( argFormatEnd != end && - *argFormatEnd != GenericSpecSymbol && // value in generic format - *argFormatEnd != 'd' && // others are standard specifiers supported by printf - *argFormatEnd != 'i' && - *argFormatEnd != 'u' && - *argFormatEnd != 'o' && - *argFormatEnd != 'x' && - *argFormatEnd != 'X' && - *argFormatEnd != 'f' && - *argFormatEnd != 'F' && - *argFormatEnd != 'e' && - *argFormatEnd != 'E' && - *argFormatEnd != 'g' && - *argFormatEnd != 'G' && - *argFormatEnd != 'a' && - *argFormatEnd != 'A' && - *argFormatEnd != 'c' && - *argFormatEnd != 's' && - *argFormatEnd != 'p' && - *argFormatEnd != 'n') + *argFormatEnd != GenericSpecSymbol && // value in generic format + !conversionSpecifiers.Contains(*argFormatEnd)) // others are standard specifiers supported by printf { switch (*argFormatEnd) { case 'q': @@ -952,27 +969,162 @@ void RunFormatter( ++argFormatEnd; } - // 'n' means 'nothing'; skip the argument. - if (*argFormatBegin != 'n') { - // Format argument. - TStringBuf argFormat(argFormatBegin, argFormatEnd); - if (singleQuotes) { - builder->AppendChar('\''); - } - if (doubleQuotes) { - builder->AppendChar('"'); - } - formatter(argIndex++, builder, argFormat); - if (singleQuotes) { - builder->AppendChar('\''); - } - if (doubleQuotes) { - builder->AppendChar('"'); + RunFormatterAt( + formatter, + argIndex++, + builder, + TStringBuf{argFormatBegin, argFormatEnd}, + singleQuotes, + doubleQuotes); + + current = argFormatEnd; + } +} + +//////////////////////////////////////////////////////////////////////////////// + +template <CFormatter TFormatter, class... TArgs> +void RunFormatter( + TStringBuilderBase* builder, + TBasicFormatString<TArgs...> formatString, + const TFormatter& formatter) +{ + auto isValidLocations = [] (const auto& t) { + return std::get<0>(t) != std::get<1>(t); + }; + // Generally marker is simply "%v" e.g. 2 symbols. + // We assume it is used to insert something for roughly 5 symbols + // of size. + builder->Preallocate(std::size(formatString.Get()) + sizeof...(TArgs) * (5 - 2)); + + // Empty marker positions -- fallback to the normal impl. + if constexpr (sizeof...(TArgs) != 0) { + if (!isValidLocations(formatString.Markers[0])) { + RunFormatterFullScan(builder, formatString.Get(), formatter); + return; + } + } else { + if (formatString.Escapes[0] == -2) { + RunFormatterFullScan(builder, formatString.Get(), formatter); + return; + } + } + + int escapesFound = 0; + int currentPos = 0; + + auto beginIt = formatString.Get().begin(); + auto size = formatString.Get().size(); + + const auto& [markers, escapes] = std::tie(formatString.Markers, formatString.Escapes); + + auto appendVerbatim = [&] (int offsetToEnd) { + builder->AppendString(TStringBuf{beginIt + currentPos, beginIt + offsetToEnd}); + }; + + auto processEscape = [&] () mutable { + // OpenMP doesn't support structured bindings :(. + auto escapePos = formatString.Escapes[escapesFound]; + + // Append everything that's present until %%. + appendVerbatim(escapePos); + + // Append '%'. + builder->AppendChar('%'); + + // Advance position to first '%' pos + 2. + currentPos = escapePos + 2; + }; + + int argIndex = 0; + + while(argIndex < std::ssize(markers)) { + auto [markerStart, markerEnd] = markers[argIndex]; + + if ( + escapes[escapesFound] != -1 && + escapes[escapesFound] - currentPos < markerStart - currentPos) + { + // Escape sequence is closer. + processEscape(); + ++escapesFound; + } else { + // Normal marker is closer. + + // Append everything that's present until marker start. + appendVerbatim(markerStart); + + // Parsing format string. + + // We skip '%' here since spec does not contain it. + auto spec = TStringBuf{beginIt + markerStart + 1, beginIt + markerEnd}; + bool singleQuotes = false; + bool doubleQuotes = false; + for (auto c : spec) { + if (c == 'q') { + singleQuotes = true; + } + if (c == 'Q') { + doubleQuotes = true; + } } + RunFormatterAt( + formatter, + argIndex, + builder, + spec, + singleQuotes, + doubleQuotes); + + // Advance position past marker's end. + currentPos = markerEnd; + ++argIndex; + continue; } - current = argFormatEnd; + // Check if the number of escapes we have found has exceeded the recorded limit + // e.g. we have to manually scan the rest of the formatString. + if (escapesFound == std::ssize(escapes)) { + break; + } } + + // Process remaining escapes. + while (escapesFound < std::ssize(escapes)) { + if (escapes[escapesFound] == -1) { + break; + } + + processEscape(); + ++escapesFound; + } + + // We either ran out of markers or reached the limit of allowed + // escape sequences. + + // Happy path: no limit on escape sequences. + if (escapesFound != std::ssize(escapes)) { + // Append whatever's left until the end. + appendVerbatim(size); + return; + } + + // Sad path: we have to fully parse remainder of format. + RunFormatterFullScan(builder, TStringBuf{beginIt + currentPos, beginIt + size}, formatter, argIndex); +} + +//////////////////////////////////////////////////////////////////////////////// + +// For benchmarking purposes. +template <class... TArgs> +TString FormatOld(TFormatString<TArgs...> format, TArgs&&... args) +{ + TStringBuilder builder; + if constexpr ((CFormattable<TArgs> && ...)) { + NYT::NDetail::TValueFormatter<0, TArgs...> formatter(args...); + NYT::NDetail::RunFormatterFullScan(&builder, format.Get(), formatter); + } + return builder.Flush(); } } // namespace NDetail @@ -991,7 +1143,7 @@ void Format(TStringBuilderBase* builder, TFormatString<TArgs...> format, TArgs&& // a second error. if constexpr ((CFormattable<TArgs> && ...)) { NYT::NDetail::TValueFormatter<0, TArgs...> formatter(args...); - NYT::NDetail::RunFormatter(builder, format.Get(), formatter); + NYT::NDetail::RunFormatter(builder, format, formatter); } } @@ -1012,7 +1164,7 @@ void FormatVector( const TVector& vec) { NYT::NDetail::TRangeFormatter<typename TVector::value_type> formatter(vec); - NYT::NDetail::RunFormatter(builder, format, formatter); + NYT::NDetail::RunFormatterFullScan(builder, format, formatter); } template <class TVector> @@ -1022,7 +1174,7 @@ void FormatVector( const TVector& vec) { NYT::NDetail::TRangeFormatter<typename TVector::value_type> formatter(vec); - NYT::NDetail::RunFormatter(builder, format, formatter); + NYT::NDetail::RunFormatterFullScan(builder, format, formatter); } template <size_t Length, class TVector> diff --git a/library/cpp/yt/string/format_analyser.h b/library/cpp/yt/string/format_analyser.h index 20eee60580..9f194144dc 100644 --- a/library/cpp/yt/string/format_analyser.h +++ b/library/cpp/yt/string/format_analyser.h @@ -4,6 +4,7 @@ #include <util/generic/strbuf.h> +#include <algorithm> #include <array> #include <string_view> @@ -14,13 +15,26 @@ namespace NYT::NDetail { struct TFormatAnalyser { public: + using TMarkerLocation = std::tuple<int, int>; + // NB(arkady-e1ppa): Location is considered invalid (e.g. not filled) + // if get<0> == get<1> == 0. + template <class... TArgs> + using TMarkerLocations = std::array<TMarkerLocation, sizeof...(TArgs)>; + // NB(arkady-e1ppa): We can't cover all of them since that would require + // dynamic storage for their coordinates and we do not have + // constexpr context large enough to deallocate dynamic memory at the + // correct time. Thus we store first 5 position and scanning afterwards + // is pessimized. |-1| is for no position at all. + // |-2| is used to imply runtime format. + using TEscapeLocations = std::array<int, 5>; + // TODO(arkady-e1ppa): Until clang-19 consteval functions // defined out of line produce symbols in rare cases // causing linker to crash. template <class... TArgs> - static consteval void ValidateFormat(std::string_view fmt) + static consteval auto AnalyzeFormat(std::string_view fmt) { - DoValidateFormat<TArgs...>(fmt); + return DoAnalyzeFormat<TArgs...>(fmt); } private: @@ -51,11 +65,16 @@ private: static constexpr char IntroductorySymbol = '%'; template <class... TArgs> - static consteval void DoValidateFormat(std::string_view format) + static consteval auto DoAnalyzeFormat(std::string_view format) { - std::array<std::string_view, sizeof...(TArgs)> markers = {}; std::array<TSpecifiers, sizeof...(TArgs)> specifiers{GetSpecifiers<TArgs>()...}; + TMarkerLocations<TArgs...> markerLocations = {}; + TEscapeLocations escapeLocations = {}; + std::ranges::fill(escapeLocations, -1); + + int escapesCount = 0; + int markerCount = 0; int currentMarkerStart = -1; @@ -81,12 +100,17 @@ private: throw "You may not terminate flag sequence other than %% with \'%\' symbol"; } // '%%' detected --- skip + if (escapesCount < std::ssize(escapeLocations)) { + escapeLocations[escapesCount] = currentMarkerStart; + ++escapesCount; + } + currentMarkerStart = -1; continue; } // We are inside of marker. - if (markerCount == std::ssize(markers)) { + if (markerCount == std::ssize(markerLocations)) { // Too many markers throw "Number of arguments supplied to format is smaller than the number of flag sequences"; } @@ -94,8 +118,8 @@ private: if (Contains(specifiers[markerCount].Conversion, symbol)) { // Marker has finished. - markers[markerCount] - = std::string_view(format.begin() + currentMarkerStart, index - currentMarkerStart + 1); + markerLocations[markerCount] + = std::tuple{currentMarkerStart, index + 1}; currentMarkerStart = -1; ++markerCount; @@ -110,16 +134,16 @@ private: if (currentMarkerStart != -1) { // Runaway marker. throw "Unterminated flag sequence detected; Use \'%%\' to type plain %"; - return; } - if (markerCount < std::ssize(markers)) { + if (markerCount < std::ssize(markerLocations)) { // Missing markers. throw "Number of arguments supplied to format is greater than the number of flag sequences"; } // TODO(arkady-e1ppa): Consider per-type verification // of markers. + return std::tuple(markerLocations, escapeLocations); } }; diff --git a/library/cpp/yt/string/format_arg.h b/library/cpp/yt/string/format_arg.h index 544e265766..4dc7be06e8 100644 --- a/library/cpp/yt/string/format_arg.h +++ b/library/cpp/yt/string/format_arg.h @@ -32,7 +32,7 @@ class TFormatArgBase public: // TODO(arkady-e1ppa): Consider more strict formatting rules. static constexpr std::array ConversionSpecifiers = { - 'v', '1', 'c', 's', 'd', 'i', 'o', + 'v', 'c', 's', 'd', 'i', 'o', 'x', 'X', 'u', 'f', 'F', 'e', 'E', 'a', 'A', 'g', 'G', 'n', 'p' }; diff --git a/library/cpp/yt/string/format_string-inl.h b/library/cpp/yt/string/format_string-inl.h index a692d9648d..67f9bad45c 100644 --- a/library/cpp/yt/string/format_string-inl.h +++ b/library/cpp/yt/string/format_string-inl.h @@ -4,6 +4,8 @@ #include "format_string.h" #endif +#include <algorithm> + namespace NYT { //////////////////////////////////////////////////////////////////////////////// @@ -15,8 +17,13 @@ consteval TBasicFormatString<TArgs...>::TBasicFormatString(const T& fmt) : Format_(fmt) { CheckFormattability(); -#if !defined(NDEBUG) && !defined(YT_DISABLE_FORMAT_STATIC_ANALYSIS) - NDetail::TFormatAnalyser::ValidateFormat<std::remove_cvref_t<TArgs>...>(Format_); +#if !defined(YT_DISABLE_FORMAT_STATIC_ANALYSIS) + std::tie(Markers, Escapes) = NDetail::TFormatAnalyser::AnalyzeFormat<std::remove_cvref_t<TArgs>...>(Format_); +#else + std::ranges::fill_n(std::ranges::begin(Escapes), 1, -1); + if constexpr (sizeof...(TArgs) != 0) { + std::ranges::fill_n(std::ranges::begin(Markers), 1, std::tuple{0, 0}); + } #endif } @@ -46,6 +53,11 @@ template <class... TArgs> TBasicFormatString<TArgs...>::TBasicFormatString(TRuntimeFormat fmt) : Format_(fmt.Get()) { + std::ranges::fill_n(std::ranges::begin(Escapes), 1, -1); + if constexpr (sizeof...(TArgs) != 0) { + std::ranges::fill_n(std::ranges::begin(Markers), 1, std::tuple{0, 0}); + } + // NB(arkady-e1ppa): StaticFormat performs the // formattability check of the args in a way // that provides more useful information diff --git a/library/cpp/yt/string/format_string.h b/library/cpp/yt/string/format_string.h index 786c2e39ed..1008ccb453 100644 --- a/library/cpp/yt/string/format_string.h +++ b/library/cpp/yt/string/format_string.h @@ -43,6 +43,10 @@ public: static consteval void CheckFormattability(); + // Data used for compile-time slicing of the format string. + NDetail::TFormatAnalyser::TMarkerLocations<TArgs...> Markers = {}; + NDetail::TFormatAnalyser::TEscapeLocations Escapes = {}; + private: std::string_view Format_; diff --git a/library/cpp/yt/string/unittests/format_ut.cpp b/library/cpp/yt/string/unittests/format_ut.cpp index ac3be99ad9..8aca8c8e29 100644 --- a/library/cpp/yt/string/unittests/format_ut.cpp +++ b/library/cpp/yt/string/unittests/format_ut.cpp @@ -326,6 +326,11 @@ TEST(TFormatTest, CustomFlagsCollectionTwoLevels) EXPECT_EQ(Format("%NRv", arr), toCollectionD2("RNP")); } +TEST(TFormatTest, ManyEscapes) +{ + EXPECT_EQ("a%b%c%d%e%f%g", Format("%v%%%v%%%v%%%v%%%v%%%v%%%g", "a", "b", "c", "d", "e", "f", "g")); +} + //////////////////////////////////////////////////////////////////////////////// } // namespace diff --git a/library/python/testing/yatest_common/yatest/common/runtime.py b/library/python/testing/yatest_common/yatest/common/runtime.py index 9d2fe5932a..e4f246d5c8 100644 --- a/library/python/testing/yatest_common/yatest/common/runtime.py +++ b/library/python/testing/yatest_common/yatest/common/runtime.py @@ -190,6 +190,8 @@ def data_path(path=None): :param path: path relative to the arcadia_tests_data directory, e.g. yatest.common.data_path("pers/rerank_service") :return: absolute path inside arcadia_tests_data """ + if "USE_ATD_FROM_SNAPSHOT" in os.environ: + return os.path.join(source_path(), "atd_ro_snapshot", path) return _join_path(_get_ya_plugin_instance().data_root, path) |