diff options
| author | Alexander Smirnov <[email protected]> | 2024-11-20 11:14:58 +0000 | 
|---|---|---|
| committer | Alexander Smirnov <[email protected]> | 2024-11-20 11:14:58 +0000 | 
| commit | 31773f157bf8164364649b5f470f52dece0a4317 (patch) | |
| tree | 33d0f7eef45303ab68cf08ab381ce5e5e36c5240 /library/cpp | |
| parent | 2c7938962d8689e175574fc1e817c05049f27905 (diff) | |
| parent | eff600952d5dfe17942f38f510a8ac2b203bb3a5 (diff) | |
Merge branch 'rightlib' into mergelibs-241120-1113
Diffstat (limited to 'library/cpp')
24 files changed, 334 insertions, 440 deletions
| diff --git a/library/cpp/lwtrace/rwspinlock.h b/library/cpp/lwtrace/rwspinlock.h index 5e4608e0683..1effd5fe0e7 100644 --- a/library/cpp/lwtrace/rwspinlock.h +++ b/library/cpp/lwtrace/rwspinlock.h @@ -1,6 +1,6 @@  #pragma once -#include <library/cpp/deprecated/atomic/atomic.h> +#include <atomic>  #include <util/system/spinlock.h> @@ -27,16 +27,16 @@  //    * writer can release lock (State = 0: -> READING)  struct TRWSpinLock { -    TAtomic State; // must be initialized by 'TRWSpinLock myLock = {0};' construction +    std::atomic_signed_lock_free State;      void Init() noexcept { -        State = 0; +        State.store(0, std::memory_order_relaxed);      }      void AcquireRead() noexcept {          while (true) { -            TAtomic a = AtomicGet(State); -            if ((a & 1) == 0 && AtomicCas(&State, a + 2, a)) { +            std::atomic_signed_lock_free::value_type a = State.load(std::memory_order_acquire); +            if ((a & 1) == 0 && State.compare_exchange_strong(a, a + 2, std::memory_order_acquire)) {                  break;              }              SpinLockPause(); @@ -44,25 +44,29 @@ struct TRWSpinLock {      }      void ReleaseRead() noexcept { -        AtomicAdd(State, -2); +        State.fetch_add(-2, std::memory_order_release);      }      void AcquireWrite() noexcept {          while (true) { -            TAtomic a = AtomicGet(State); -            if ((a & 1) == 0 && AtomicCas(&State, a + 1, a)) { +            std::atomic_signed_lock_free::value_type a = State.load(std::memory_order_acquire); +            if ((a & 1) == 0 && State.compare_exchange_strong(a, a + 1, std::memory_order_acquire)) {                  break;              }              SpinLockPause();          } -        while (!AtomicCas(&State, TAtomicBase(-1), 1)) { +        while (true) { +            std::atomic_signed_lock_free::value_type a = 1; +            if (State.compare_exchange_strong(a, -1, std::memory_order_acquire)) { +                break; +            }              SpinLockPause();          }      }      void ReleaseWrite() noexcept { -        AtomicSet(State, 0); +        State.store(0, std::memory_order_release);      }  }; diff --git a/library/cpp/netliba/v6/udp_address.cpp b/library/cpp/netliba/v6/udp_address.cpp index ba5fd6f45b1..be823e2dfa3 100644 --- a/library/cpp/netliba/v6/udp_address.cpp +++ b/library/cpp/netliba/v6/udp_address.cpp @@ -196,7 +196,7 @@ namespace NNetliba {                  continue;              }              if (ptr->Mtu < 1280) { -                fprintf(stderr, "WARNING: MTU %d is less then ipv6 minimum", ptr->Mtu); +                fprintf(stderr, "WARNING: MTU %lu is less then ipv6 minimum", ptr->Mtu);              }              for (IP_ADAPTER_UNICAST_ADDRESS* addr = ptr->FirstUnicastAddress; addr; addr = addr->Next) {                  sockaddr* x = (sockaddr*)addr->Address.lpSockaddr; diff --git a/library/cpp/pop_count/benchmark/main.cpp b/library/cpp/pop_count/benchmark/main.cpp deleted file mode 100644 index 1af2d157b61..00000000000 --- a/library/cpp/pop_count/benchmark/main.cpp +++ /dev/null @@ -1,82 +0,0 @@ -#include <util/stream/output.h> -#include <util/datetime/cputimer.h> -#include <util/system/type_name.h> - -#include <library/cpp/pop_count/popcount.h> -#include <library/cpp/testing/benchmark/bench.h> - -#include <bit> - -template <class F, class I> -inline void DoRun(F&& f, I&& i) { -    const ui64 n = i.Iterations(); - -    for (ui64 j = 0; j < n; ++j) { -        Y_DO_NOT_OPTIMIZE_AWAY(f(j * (ui64)123456 + (ui64)1)); -    } -} - -Y_CPU_BENCHMARK(PopCount_8, iface) { -    DoRun([](ui8 x) { -        return PopCount<ui8>(x); -    }, -          iface); -} - -Y_CPU_BENCHMARK(std_popcount_8, iface) { -    DoRun([](ui8 x) { -        return std::popcount<ui8>(x); -    }, -          iface); -} - -Y_CPU_BENCHMARK(PopCount_16, iface) { -    DoRun([](ui16 x) { -        return PopCount<ui16>(x); -    }, -          iface); -} - -Y_CPU_BENCHMARK(std_popcount_16, iface) { -    DoRun([](ui16 x) { -        return std::popcount<ui16>(x); -    }, -          iface); -} - -Y_CPU_BENCHMARK(PopCount_32, iface) { -    DoRun([](ui32 x) { -        return PopCount<ui32>(x); -    }, -          iface); -} - -Y_CPU_BENCHMARK(std_popcount_32, iface) { -    DoRun([](ui32 x) { -        return std::popcount<ui32>(x); -    }, -          iface); -} - -Y_CPU_BENCHMARK(PopCount_64, iface) { -    DoRun([](ui64 x) { -        return PopCount<ui64>(x); -    }, -          iface); -} - -Y_CPU_BENCHMARK(std_popcount_64, iface) { -    DoRun([](ui64 x) { -        return std::popcount<ui64>(x); -    }, -          iface); -} - -#if !defined(_MSC_VER) -Y_CPU_BENCHMARK(BUILTIN_64, iface) { -    DoRun([](ui64 x) { -        return __builtin_popcountll(x); -    }, -          iface); -} -#endif diff --git a/library/cpp/pop_count/benchmark/ya.make b/library/cpp/pop_count/benchmark/ya.make deleted file mode 100644 index e7ea9418db5..00000000000 --- a/library/cpp/pop_count/benchmark/ya.make +++ /dev/null @@ -1,12 +0,0 @@ -Y_BENCHMARK() - -PEERDIR( -    util/draft -    library/cpp/pop_count -) - -SRCS( -    main.cpp -) - -END() diff --git a/library/cpp/pop_count/popcount.cpp b/library/cpp/pop_count/popcount.cpp deleted file mode 100644 index 49276424be1..00000000000 --- a/library/cpp/pop_count/popcount.cpp +++ /dev/null @@ -1,30 +0,0 @@ -#include "popcount.h" - -#include <util/system/defaults.h> -#include <util/system/yassert.h> - -#include <string.h> - -static const ui8 PopCountLUT8Impl[1 << 8] = { -#define B2(n) n, n + 1, n + 1, n + 2 -#define B4(n) B2(n), B2(n + 1), B2(n + 1), B2(n + 2) -#define B6(n) B4(n), B4(n + 1), B4(n + 1), B4(n + 2) -    B6(0), B6(1), B6(1), B6(2)}; - -ui8 const* PopCountLUT8 = PopCountLUT8Impl; - -#if !defined(_MSC_VER) -//ICE here for msvc - -static const ui8 PopCountLUT16Impl[1 << 16] = { -#define B2(n) n, n + 1, n + 1, n + 2 -#define B4(n) B2(n), B2(n + 1), B2(n + 1), B2(n + 2) -#define B6(n) B4(n), B4(n + 1), B4(n + 1), B4(n + 2) -#define B8(n) B6(n), B6(n + 1), B6(n + 1), B6(n + 2) -#define B10(n) B8(n), B8(n + 1), B8(n + 1), B8(n + 2) -#define B12(n) B10(n), B10(n + 1), B10(n + 1), B10(n + 2) -#define B14(n) B12(n), B12(n + 1), B12(n + 1), B12(n + 2) -    B14(0), B14(1), B14(1), B14(2)}; - -ui8 const* PopCountLUT16 = PopCountLUT16Impl; -#endif diff --git a/library/cpp/pop_count/popcount.h b/library/cpp/pop_count/popcount.h deleted file mode 100644 index 3d67737ed25..00000000000 --- a/library/cpp/pop_count/popcount.h +++ /dev/null @@ -1,105 +0,0 @@ -#pragma once - -#include <util/generic/typelist.h> -#include <util/system/cpu_id.h> -#include <util/system/defaults.h> -#include <util/system/hi_lo.h> -#include <util/system/platform.h> - -#if defined(_MSC_VER) -#include <intrin.h> -#endif - -static inline ui32 PopCountImpl(ui8 n) { -#if defined(_ppc64_) -    ui32 r; -    __asm__("popcntb %0, %1" -            : "=r"(r) -            : "r"(n) -            :); -    return r; -#else -    extern ui8 const* PopCountLUT8; -    return PopCountLUT8[n]; -#endif -} - -static inline ui32 PopCountImpl(ui16 n) { -#if defined(_MSC_VER) -    return __popcnt16(n); -#else -    extern ui8 const* PopCountLUT16; -    return PopCountLUT16[n]; -#endif -} - -static inline ui32 PopCountImpl(ui32 n) { -#if defined(_MSC_VER) -    return __popcnt(n); -#else -#if defined(_x86_64_) -    if (NX86::CachedHavePOPCNT()) { -        ui32 r; - -        __asm__("popcnt %1, %0;" -                : "=r"(r) -                : "r"(n) -                :); - -        return r; -    } -#else -#if defined(_ppc64_) -    ui32 r; - -    __asm__("popcntw %0, %1" -            : "=r"(r) -            : "r"(n) -            :); - -    return r; -#endif -#endif - -    return PopCountImpl((ui16)Lo16(n)) + PopCountImpl((ui16)Hi16(n)); -#endif -} - -static inline ui32 PopCountImpl(ui64 n) { -#if defined(_MSC_VER) && !defined(_i386_) -    return __popcnt64(n); -#else -#if defined(_x86_64_) -    if (NX86::CachedHavePOPCNT()) { -        ui64 r; - -        __asm__("popcnt %1, %0;" -                : "=r"(r) -                : "r"(n) -                :); - -        return r; -    } -#else -#if defined(_ppc64_) -    ui32 r; - -    __asm__("popcntd %0, %1" -            : "=r"(r) -            : "r"(n) -            :); - -    return r; -#endif -#endif - -    return PopCountImpl((ui32)Lo32(n)) + PopCountImpl((ui32)Hi32(n)); -#endif -} - -template <class T> -static inline ui32 PopCount(T n) { -    using TCvt = TFixedWidthUnsignedInt<T>; - -    return PopCountImpl((TCvt)n); -} diff --git a/library/cpp/pop_count/popcount_ut.cpp b/library/cpp/pop_count/popcount_ut.cpp deleted file mode 100644 index 5cd6605411f..00000000000 --- a/library/cpp/pop_count/popcount_ut.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include "popcount.h" - -#include <library/cpp/testing/unittest/registar.h> - -#include <util/random/random.h> - -Y_UNIT_TEST_SUITE(TestPopCount) { -    template <class T> -    static inline ui32 SlowPopCount(T t) { -        ui32 ret = 0; - -        while (t) { -            if (t & T(1)) { -                ++ret; -            } - -            t = t >> 1; -        } - -        return ret; -    } - -    template <class T> -    static inline void Test() { -        for (size_t i = 0; i < 10000; ++i) { -            const T rndv = RandomNumber<T>(); - -            UNIT_ASSERT_VALUES_EQUAL(SlowPopCount(rndv), PopCount(rndv)); -        } -    } - -    Y_UNIT_TEST(Test8) { -        Test<ui8>(); -    } - -    Y_UNIT_TEST(Test16) { -        Test<ui16>(); -    } - -    Y_UNIT_TEST(Test32) { -        Test<ui32>(); -    } - -    Y_UNIT_TEST(Test64) { -        Test<ui64>(); -    } - -    Y_UNIT_TEST(TestPopCount) { -        UNIT_ASSERT_VALUES_EQUAL(PopCount(0), 0); -        UNIT_ASSERT_VALUES_EQUAL(PopCount(1), 1); -        UNIT_ASSERT_VALUES_EQUAL(PopCount(1 << 10), 1); -        UNIT_ASSERT_VALUES_EQUAL(PopCount((1 << 10) + 1), 2); -        UNIT_ASSERT_VALUES_EQUAL(PopCount(0xFFFF), 16); -        UNIT_ASSERT_VALUES_EQUAL(PopCount(0xFFFFFFFF), 32); -        UNIT_ASSERT_VALUES_EQUAL(PopCount(0x55555555), 16); - -        UNIT_ASSERT_VALUES_EQUAL(0, PopCount(0ULL)); -        UNIT_ASSERT_VALUES_EQUAL(1, PopCount(1ULL)); -        UNIT_ASSERT_VALUES_EQUAL(16, PopCount(0xAAAAAAAAULL)); -        UNIT_ASSERT_VALUES_EQUAL(32, PopCount(0xFFFFFFFFULL)); -        UNIT_ASSERT_VALUES_EQUAL(32, PopCount(0xAAAAAAAAAAAAAAAAULL)); -        UNIT_ASSERT_VALUES_EQUAL(64, PopCount(0xFFFFFFFFFFFFFFFFULL)); - -        ui64 v = 0; - -        for (int i = 0; i < 64; v |= 1ULL << i, ++i) { -            UNIT_ASSERT_VALUES_EQUAL(i, PopCount(v)); -        } -    } -} diff --git a/library/cpp/pop_count/ut/ya.make b/library/cpp/pop_count/ut/ya.make deleted file mode 100644 index 179d47f064d..00000000000 --- a/library/cpp/pop_count/ut/ya.make +++ /dev/null @@ -1,7 +0,0 @@ -UNITTEST_FOR(library/cpp/pop_count) - -SRCS( -    popcount_ut.cpp -) - -END() diff --git a/library/cpp/pop_count/ya.make b/library/cpp/pop_count/ya.make deleted file mode 100644 index d858f3e87ec..00000000000 --- a/library/cpp/pop_count/ya.make +++ /dev/null @@ -1,12 +0,0 @@ -LIBRARY() - -SRCS( -    popcount.cpp -) - -END() - -RECURSE( -    benchmark -    ut -) diff --git a/library/cpp/string_utils/ztstrbuf/ztstrbuf_ut.cpp b/library/cpp/string_utils/ztstrbuf/ztstrbuf_ut.cpp index d8215abdfad..b22d4bf216d 100644 --- a/library/cpp/string_utils/ztstrbuf/ztstrbuf_ut.cpp +++ b/library/cpp/string_utils/ztstrbuf/ztstrbuf_ut.cpp @@ -12,7 +12,7 @@ Y_UNIT_TEST_SUITE(TZtStringBufTest) {      Y_UNIT_TEST(Constness) {          constexpr TZtStringBuf s0{"bar"};          static_assert(s0[0] == 'b'); -        static_assert(s0[3] == '\0'); +        static_assert(s0.data()[s0.size()] == '\0');          static_assert(s0.data()[2] == 'r');          UNIT_ASSERT_VALUES_EQUAL(s0, TString{"bar"});      } diff --git a/library/cpp/svnversion/ya.make b/library/cpp/svnversion/ya.make index d028f45c94d..27e91546445 100644 --- a/library/cpp/svnversion/ya.make +++ b/library/cpp/svnversion/ya.make @@ -6,7 +6,7 @@ SRCS(  )  IF (OPENSOURCE_PROJECT == "yt-cpp-sdk") -    PEERDIR(build/scripts/c_templates/) +    PEERDIR(build/scripts/c_templates)  ENDIF()  END() diff --git a/library/cpp/testing/common/env.cpp b/library/cpp/testing/common/env.cpp index 13f3f186afd..67e081c371c 100644 --- a/library/cpp/testing/common/env.cpp +++ b/library/cpp/testing/common/env.cpp @@ -41,6 +41,10 @@ TString BinaryPath(TStringBuf path) {  }  TString GetArcadiaTestsData() { +    if (GetEnv("USE_ATD_FROM_SNAPSHOT")) { +        return ArcadiaSourceRoot() + "/atd_ro_snapshot"; +    } +      TString atdRoot = NPrivate::GetTestEnv().ArcadiaTestsDataDir;      if (atdRoot) {          return atdRoot; diff --git a/library/cpp/testing/mock_server/ut/server_ut.cpp b/library/cpp/testing/mock_server/ut/server_ut.cpp deleted file mode 100644 index 74b3dcec050..00000000000 --- a/library/cpp/testing/mock_server/ut/server_ut.cpp +++ /dev/null @@ -1,40 +0,0 @@ -#include <library/cpp/testing/mock_server/server.h> - -#include <library/cpp/http/simple/http_client.h> -#include <library/cpp/testing/unittest/registar.h> - -Y_UNIT_TEST_SUITE(Server) { -    int i; - -    Y_UNIT_TEST(pong) { -        TPortManager pm; -        ui16 port = pm.GetPort(80); -        NMock::TMockServer server(port, []() { return new NMock::TPong; }); - -        TKeepAliveHttpClient cl("localhost", port); -        UNIT_ASSERT_VALUES_EQUAL(200, cl.DoGet("/ping")); -        UNIT_ASSERT_VALUES_EQUAL(404, cl.DoGet("/kek")); -    } - -    Y_UNIT_TEST(custom) { -        class TCustomReplier: public TRequestReplier { -        public: -            bool DoReply(const TReplyParams& params) override { -                THttpResponse resp(HttpCodes::HTTP_OK); -                resp.SetContent("everithing is ok"); -                resp.OutTo(params.Output); - -                return true; -            } -        }; - -        TPortManager pm; -        ui16 port = pm.GetPort(80); -        NMock::TMockServer server(port, []() { return new TCustomReplier; }); - -        TKeepAliveHttpClient cl("localhost", port); -        TStringStream out; -        UNIT_ASSERT_VALUES_EQUAL(200, cl.DoGet("/foo", &out)); -        UNIT_ASSERT_VALUES_EQUAL("everithing is ok", out.Str()); -    } -} diff --git a/library/cpp/testing/mock_server/ut/ya.make b/library/cpp/testing/mock_server/ut/ya.make deleted file mode 100644 index f4ee34458fc..00000000000 --- a/library/cpp/testing/mock_server/ut/ya.make +++ /dev/null @@ -1,11 +0,0 @@ -UNITTEST_FOR(library/cpp/testing/mock_server) - -PEERDIR( -    library/cpp/http/simple -) - -SRCS( -    server_ut.cpp -) - -END() diff --git a/library/cpp/tld/tlds-alpha-by-domain.txt b/library/cpp/tld/tlds-alpha-by-domain.txt index 75edec6f550..dc6477cc73f 100644 --- a/library/cpp/tld/tlds-alpha-by-domain.txt +++ b/library/cpp/tld/tlds-alpha-by-domain.txt @@ -1,4 +1,4 @@ -# Version 2024111300, Last Updated Wed Nov 13 07:07:01 2024 UTC +# Version 2024111900, Last Updated Tue Nov 19 07:07:01 2024 UTC  AAA  AARP  ABB diff --git a/library/cpp/yt/misc/cast-inl.h b/library/cpp/yt/misc/cast-inl.h index 50920f01934..5e5a0d90eea 100644 --- a/library/cpp/yt/misc/cast-inl.h +++ b/library/cpp/yt/misc/cast-inl.h @@ -119,7 +119,7 @@ T CheckedIntegralCast(S value)  template <class T, class S>      requires TEnumTraits<T>::IsEnum -constexpr std::optional<T> TryCheckedEnumCast(S value) +constexpr std::optional<T> TryCheckedEnumCast(S value, bool enableUnknown)  {      auto underlying = TryCheckedIntegralCast<std::underlying_type_t<T>>(value);      [[unlikely]] if (!underlying) { @@ -127,6 +127,15 @@ constexpr std::optional<T> TryCheckedEnumCast(S value)      }      auto candidate = static_cast<T>(*underlying);      [[unlikely]] if (!TEnumTraits<T>::IsValidValue(candidate)) { +        if (enableUnknown) { +            if constexpr (constexpr auto optionalUnknownValue = TEnumTraits<T>::TryGetUnknownValue()) { +                if constexpr (TEnumTraits<T>::IsBitEnum) { +                    return static_cast<T>(*underlying & ToUnderlying(TEnumTraits<T>::GetAllSetValue())) | *optionalUnknownValue; +                } else { +                    return *optionalUnknownValue; +                } +            } +        }          return std::nullopt;      }      return candidate; @@ -136,7 +145,7 @@ template <class T, class S>      requires TEnumTraits<T>::IsEnum  T CheckedEnumCast(S value)  { -    auto result = TryCheckedEnumCast<T>(value); +    auto result = TryCheckedEnumCast<T>(value, /*enableUnknown*/ true);      [[unlikely]] if (!result) {          if constexpr (std::is_signed_v<S>) {              throw TSimpleException(Sprintf("Error casting %s value \"%" PRIi64 "\" to enum %s", diff --git a/library/cpp/yt/misc/cast.h b/library/cpp/yt/misc/cast.h index a4e32e0287d..ccf1eb21a6b 100644 --- a/library/cpp/yt/misc/cast.h +++ b/library/cpp/yt/misc/cast.h @@ -26,7 +26,7 @@ T CheckedIntegralCast(S value);  template <class T, class S>      requires TEnumTraits<T>::IsEnum -constexpr std::optional<T> TryCheckedEnumCast(S value); +constexpr std::optional<T> TryCheckedEnumCast(S value, bool enableUnknown = false);  template <class T, class S>      requires TEnumTraits<T>::IsEnum diff --git a/library/cpp/yt/misc/unittests/cast_ut.cpp b/library/cpp/yt/misc/unittests/cast_ut.cpp index 8662575d6c0..e6f5409f2ff 100644 --- a/library/cpp/yt/misc/unittests/cast_ut.cpp +++ b/library/cpp/yt/misc/unittests/cast_ut.cpp @@ -21,6 +21,26 @@ DEFINE_BIT_ENUM_WITH_UNDERLYING_TYPE(EFeatures, ui8,      ((Second)(0x0002))  ); +DEFINE_BIT_ENUM(ELangsWithUnknown, +    ((None)       (0x00)) +    ((Cpp)        (0x01)) +    ((Go)         (0x02)) +    ((Rust)       (0x04)) +    ((Python)     (0x08)) +    ((JavaScript) (0x10)) +    ((CppGo)      (0x03)) +    ((All)        (0x1f)) +    ((Unknown)    (0x20)) +); +DEFINE_ENUM_UNKNOWN_VALUE(ELangsWithUnknown, Unknown); + +DEFINE_ENUM(EColorWithUnknown, +    ((Red)     (0)) +    ((Green)   (1)) +    ((Unknown) (2)) +); +DEFINE_ENUM_UNKNOWN_VALUE(EColorWithUnknown, Unknown); +  TEST(TCastTest, TryCheckedEnumCast)  {      EXPECT_EQ((TryCheckedEnumCast<ECardinal, char>(2)), ECardinal::East); @@ -36,6 +56,35 @@ TEST(TCastTest, TryCheckedEnumCast)      EXPECT_EQ((TryCheckedEnumCast<EFeatures, ui8>(ToUnderlying(EFeatures::First | EFeatures::Second))), EFeatures::First | EFeatures::Second);      EXPECT_FALSE((TryCheckedEnumCast<EFeatures, ui8>(0x10))); + +    EXPECT_FALSE(TryCheckedEnumCast<EColorWithUnknown>(3)); +    EXPECT_EQ(TryCheckedEnumCast<EColorWithUnknown>(3, /*enableUnknown*/ true), EColorWithUnknown::Unknown); + +    EXPECT_FALSE(TryCheckedEnumCast<ELangsWithUnknown>(0x40)); +    EXPECT_EQ(TryCheckedEnumCast<ELangsWithUnknown>(0x40, /*enableUnknown*/ true), ELangsWithUnknown::Unknown); +    EXPECT_EQ(TryCheckedEnumCast<ELangsWithUnknown>(0x41, /*enableUnknown*/ true), ELangsWithUnknown::Unknown | ELangsWithUnknown::Cpp); +} + +TEST(TCastTest, CheckedEnumCast) +{ +    EXPECT_EQ((CheckedEnumCast<ECardinal, char>(2)), ECardinal::East); +    EXPECT_EQ((CheckedEnumCast<ECardinal, int>(3)), ECardinal::South); + +    EXPECT_THROW((CheckedEnumCast<ECardinal, char>(100)), TSimpleException); +    EXPECT_THROW((CheckedEnumCast<ECardinal, int>(300)), TSimpleException); + +    EXPECT_EQ((CheckedEnumCast<EFeatures, ui8>(0)), EFeatures::None); +    EXPECT_EQ((CheckedEnumCast<EFeatures, ui8>(ToUnderlying(EFeatures::First))), EFeatures::First); +    EXPECT_EQ((CheckedEnumCast<EFeatures, ui8>(ToUnderlying(EFeatures::Second))), EFeatures::Second); +    EXPECT_EQ((CheckedEnumCast<EFeatures, int>(ToUnderlying(EFeatures::First))), EFeatures::First); +    EXPECT_EQ((CheckedEnumCast<EFeatures, ui8>(ToUnderlying(EFeatures::First | EFeatures::Second))), EFeatures::First | EFeatures::Second); + +    EXPECT_THROW((CheckedEnumCast<EFeatures, ui8>(0x10)), TSimpleException); + +    EXPECT_EQ(CheckedEnumCast<EColorWithUnknown>(3), EColorWithUnknown::Unknown); + +    EXPECT_EQ(CheckedEnumCast<ELangsWithUnknown>(0x40), ELangsWithUnknown::Unknown); +    EXPECT_EQ(CheckedEnumCast<ELangsWithUnknown>(0x41), ELangsWithUnknown::Unknown | ELangsWithUnknown::Cpp);  }  //////////////////////////////////////////////////////////////////////////////// diff --git a/library/cpp/yt/string/format-inl.h b/library/cpp/yt/string/format-inl.h index 7f1f7259665..e90d68bfe12 100644 --- a/library/cpp/yt/string/format-inl.h +++ b/library/cpp/yt/string/format-inl.h @@ -871,12 +871,43 @@ concept CFormatter = CInvocable<T, void(size_t, TStringBuilderBase*, TStringBuf)  ////////////////////////////////////////////////////////////////////////////////  template <CFormatter TFormatter> -void RunFormatter( +void RunFormatterAt( +    const TFormatter& formatter, +    size_t index, +    TStringBuilderBase* builder, +    TStringBuf spec, +    bool singleQuotes, +    bool doubleQuotes) +{ +    // 'n' means 'nothing'; skip the argument. +    if (!spec.Contains('n')) { +        if (singleQuotes) { +            builder->AppendChar('\''); +        } +        if (doubleQuotes) { +            builder->AppendChar('"'); +        } + +        formatter(index, builder, spec); + +        if (singleQuotes) { +            builder->AppendChar('\''); +        } +        if (doubleQuotes) { +            builder->AppendChar('"'); +        } +    } +} + +//////////////////////////////////////////////////////////////////////////////// + +template <CFormatter TFormatter> +void RunFormatterFullScan(      TStringBuilderBase* builder,      TStringBuf format, -    const TFormatter& formatter) +    const TFormatter& formatter, +    int argIndex = 0)  { -    size_t argIndex = 0;      auto current = std::begin(format);      auto end = std::end(format);      while (true) { @@ -912,27 +943,13 @@ void RunFormatter(          bool singleQuotes = false;          bool doubleQuotes = false; +        static constexpr TStringBuf conversionSpecifiers = +            "diuoxXfFeEgGaAcspn"; +          while (              argFormatEnd != end && -            *argFormatEnd != GenericSpecSymbol &&     // value in generic format -            *argFormatEnd != 'd' &&                   // others are standard specifiers supported by printf -            *argFormatEnd != 'i' && -            *argFormatEnd != 'u' && -            *argFormatEnd != 'o' && -            *argFormatEnd != 'x' && -            *argFormatEnd != 'X' && -            *argFormatEnd != 'f' && -            *argFormatEnd != 'F' && -            *argFormatEnd != 'e' && -            *argFormatEnd != 'E' && -            *argFormatEnd != 'g' && -            *argFormatEnd != 'G' && -            *argFormatEnd != 'a' && -            *argFormatEnd != 'A' && -            *argFormatEnd != 'c' && -            *argFormatEnd != 's' && -            *argFormatEnd != 'p' && -            *argFormatEnd != 'n') +            *argFormatEnd != GenericSpecSymbol &&            // value in generic format +            !conversionSpecifiers.Contains(*argFormatEnd)) // others are standard specifiers supported by printf          {              switch (*argFormatEnd) {                  case 'q': @@ -952,27 +969,162 @@ void RunFormatter(              ++argFormatEnd;          } -        // 'n' means 'nothing'; skip the argument. -        if (*argFormatBegin != 'n') { -            // Format argument. -            TStringBuf argFormat(argFormatBegin, argFormatEnd); -            if (singleQuotes) { -                builder->AppendChar('\''); -            } -            if (doubleQuotes) { -                builder->AppendChar('"'); -            } -            formatter(argIndex++, builder, argFormat); -            if (singleQuotes) { -                builder->AppendChar('\''); -            } -            if (doubleQuotes) { -                builder->AppendChar('"'); +        RunFormatterAt( +            formatter, +            argIndex++, +            builder, +            TStringBuf{argFormatBegin, argFormatEnd}, +            singleQuotes, +            doubleQuotes); + +        current = argFormatEnd; +    } +} + +//////////////////////////////////////////////////////////////////////////////// + +template <CFormatter TFormatter, class... TArgs> +void RunFormatter( +    TStringBuilderBase* builder, +    TBasicFormatString<TArgs...> formatString, +    const TFormatter& formatter) +{ +    auto isValidLocations = [] (const auto& t) { +        return std::get<0>(t) != std::get<1>(t); +    }; +    // Generally marker is simply "%v" e.g. 2 symbols. +    // We assume it is used to insert something for roughly 5 symbols +    // of size. +    builder->Preallocate(std::size(formatString.Get()) + sizeof...(TArgs) * (5 - 2)); + +    // Empty marker positions -- fallback to the normal impl. +    if constexpr (sizeof...(TArgs) != 0) { +        if (!isValidLocations(formatString.Markers[0])) { +            RunFormatterFullScan(builder, formatString.Get(), formatter); +            return; +        } +    } else { +        if (formatString.Escapes[0] == -2) { +            RunFormatterFullScan(builder, formatString.Get(), formatter); +            return; +        } +    } + +    int escapesFound = 0; +    int currentPos = 0; + +    auto beginIt = formatString.Get().begin(); +    auto size = formatString.Get().size(); + +    const auto& [markers, escapes] = std::tie(formatString.Markers, formatString.Escapes); + +    auto appendVerbatim = [&] (int offsetToEnd) { +        builder->AppendString(TStringBuf{beginIt + currentPos, beginIt + offsetToEnd}); +    }; + +    auto processEscape = [&] () mutable { +        // OpenMP doesn't support structured bindings :(. +        auto escapePos = formatString.Escapes[escapesFound]; + +        // Append everything that's present until %%. +        appendVerbatim(escapePos); + +        // Append '%'. +        builder->AppendChar('%'); + +        // Advance position to first '%' pos + 2. +        currentPos = escapePos + 2; +    }; + +    int argIndex = 0; + +    while(argIndex < std::ssize(markers)) { +        auto [markerStart, markerEnd] = markers[argIndex]; + +        if ( +            escapes[escapesFound] != -1 && +            escapes[escapesFound] - currentPos < markerStart - currentPos) +        { +            // Escape sequence is closer. +            processEscape(); +            ++escapesFound; +        } else { +            // Normal marker is closer. + +            // Append everything that's present until marker start. +            appendVerbatim(markerStart); + +            // Parsing format string. + +            // We skip '%' here since spec does not contain it. +            auto spec = TStringBuf{beginIt + markerStart + 1, beginIt + markerEnd}; +            bool singleQuotes = false; +            bool doubleQuotes = false; +            for (auto c : spec) { +                if (c == 'q') { +                    singleQuotes = true; +                } +                if (c == 'Q') { +                    doubleQuotes = true; +                }              } +            RunFormatterAt( +                formatter, +                argIndex, +                builder, +                spec, +                singleQuotes, +                doubleQuotes); + +            // Advance position past marker's end. +            currentPos = markerEnd; +            ++argIndex; +            continue;          } -        current = argFormatEnd; +        // Check if the number of escapes we have found has exceeded the recorded limit +        // e.g. we have to manually scan the rest of the formatString. +        if (escapesFound == std::ssize(escapes)) { +            break; +        }      } + +    // Process remaining escapes. +    while (escapesFound < std::ssize(escapes)) { +        if (escapes[escapesFound] == -1) { +            break; +        } + +        processEscape(); +        ++escapesFound; +    } + +    // We either ran out of markers or reached the limit of allowed +    // escape sequences. + +    // Happy path: no limit on escape sequences. +    if (escapesFound != std::ssize(escapes)) { +        // Append whatever's left until the end. +        appendVerbatim(size); +        return; +    } + +    // Sad path: we have to fully parse remainder of format. +    RunFormatterFullScan(builder, TStringBuf{beginIt + currentPos, beginIt + size}, formatter, argIndex); +} + +//////////////////////////////////////////////////////////////////////////////// + +// For benchmarking purposes. +template <class... TArgs> +TString FormatOld(TFormatString<TArgs...> format, TArgs&&... args) +{ +    TStringBuilder builder; +    if constexpr ((CFormattable<TArgs> && ...)) { +        NYT::NDetail::TValueFormatter<0, TArgs...> formatter(args...); +        NYT::NDetail::RunFormatterFullScan(&builder, format.Get(), formatter); +    } +    return builder.Flush();  }  } // namespace NDetail @@ -991,7 +1143,7 @@ void Format(TStringBuilderBase* builder, TFormatString<TArgs...> format, TArgs&&      // a second error.      if constexpr ((CFormattable<TArgs> && ...)) {          NYT::NDetail::TValueFormatter<0, TArgs...> formatter(args...); -        NYT::NDetail::RunFormatter(builder, format.Get(), formatter); +        NYT::NDetail::RunFormatter(builder, format, formatter);      }  } @@ -1012,7 +1164,7 @@ void FormatVector(      const TVector& vec)  {      NYT::NDetail::TRangeFormatter<typename TVector::value_type> formatter(vec); -    NYT::NDetail::RunFormatter(builder, format, formatter); +    NYT::NDetail::RunFormatterFullScan(builder, format, formatter);  }  template <class TVector> @@ -1022,7 +1174,7 @@ void FormatVector(      const TVector& vec)  {      NYT::NDetail::TRangeFormatter<typename TVector::value_type> formatter(vec); -    NYT::NDetail::RunFormatter(builder, format, formatter); +    NYT::NDetail::RunFormatterFullScan(builder, format, formatter);  }  template <size_t Length, class TVector> diff --git a/library/cpp/yt/string/format_analyser.h b/library/cpp/yt/string/format_analyser.h index 20eee605809..9f194144dcb 100644 --- a/library/cpp/yt/string/format_analyser.h +++ b/library/cpp/yt/string/format_analyser.h @@ -4,6 +4,7 @@  #include <util/generic/strbuf.h> +#include <algorithm>  #include <array>  #include <string_view> @@ -14,13 +15,26 @@ namespace NYT::NDetail {  struct TFormatAnalyser  {  public: +    using TMarkerLocation = std::tuple<int, int>; +    // NB(arkady-e1ppa): Location is considered invalid (e.g. not filled) +    // if get<0> == get<1> == 0. +    template <class... TArgs> +    using TMarkerLocations = std::array<TMarkerLocation, sizeof...(TArgs)>; +    // NB(arkady-e1ppa): We can't cover all of them since that would require +    // dynamic storage for their coordinates and we do not have +    // constexpr context large enough to deallocate dynamic memory at the +    // correct time. Thus we store first 5 position and scanning afterwards +    // is pessimized. |-1| is for no position at all. +    // |-2| is used to imply runtime format. +    using TEscapeLocations = std::array<int, 5>; +      // TODO(arkady-e1ppa): Until clang-19 consteval functions      // defined out of line produce symbols in rare cases      // causing linker to crash.      template <class... TArgs> -    static consteval void ValidateFormat(std::string_view fmt) +    static consteval auto AnalyzeFormat(std::string_view fmt)      { -        DoValidateFormat<TArgs...>(fmt); +        return DoAnalyzeFormat<TArgs...>(fmt);      }  private: @@ -51,11 +65,16 @@ private:      static constexpr char IntroductorySymbol = '%';      template <class... TArgs> -    static consteval void DoValidateFormat(std::string_view format) +    static consteval auto DoAnalyzeFormat(std::string_view format)      { -        std::array<std::string_view, sizeof...(TArgs)> markers = {};          std::array<TSpecifiers, sizeof...(TArgs)> specifiers{GetSpecifiers<TArgs>()...}; +        TMarkerLocations<TArgs...> markerLocations = {}; +        TEscapeLocations escapeLocations = {}; +        std::ranges::fill(escapeLocations, -1); + +        int escapesCount = 0; +          int markerCount = 0;          int currentMarkerStart = -1; @@ -81,12 +100,17 @@ private:                      throw "You may not terminate flag sequence other than %% with \'%\' symbol";                  }                  // '%%' detected --- skip +                if (escapesCount < std::ssize(escapeLocations)) { +                    escapeLocations[escapesCount] = currentMarkerStart; +                    ++escapesCount; +                } +                  currentMarkerStart = -1;                  continue;              }              // We are inside of marker. -            if (markerCount == std::ssize(markers)) { +            if (markerCount == std::ssize(markerLocations)) {                  // Too many markers                  throw "Number of arguments supplied to format is smaller than the number of flag sequences";              } @@ -94,8 +118,8 @@ private:              if (Contains(specifiers[markerCount].Conversion, symbol)) {                  // Marker has finished. -                markers[markerCount] -                    = std::string_view(format.begin() + currentMarkerStart, index - currentMarkerStart + 1); +                markerLocations[markerCount] +                    = std::tuple{currentMarkerStart, index + 1};                  currentMarkerStart = -1;                  ++markerCount; @@ -110,16 +134,16 @@ private:          if (currentMarkerStart != -1) {              // Runaway marker.              throw "Unterminated flag sequence detected; Use \'%%\' to type plain %"; -            return;          } -        if (markerCount < std::ssize(markers)) { +        if (markerCount < std::ssize(markerLocations)) {              // Missing markers.              throw "Number of arguments supplied to format is greater than the number of flag sequences";          }          // TODO(arkady-e1ppa): Consider per-type verification          // of markers. +        return std::tuple(markerLocations, escapeLocations);      }  }; diff --git a/library/cpp/yt/string/format_arg.h b/library/cpp/yt/string/format_arg.h index 544e2657667..4dc7be06e8f 100644 --- a/library/cpp/yt/string/format_arg.h +++ b/library/cpp/yt/string/format_arg.h @@ -32,7 +32,7 @@ class TFormatArgBase  public:      // TODO(arkady-e1ppa): Consider more strict formatting rules.      static constexpr std::array ConversionSpecifiers = { -        'v', '1', 'c', 's', 'd', 'i', 'o', +        'v', 'c', 's', 'd', 'i', 'o',          'x', 'X', 'u', 'f', 'F', 'e', 'E',          'a', 'A', 'g', 'G', 'n', 'p'      }; diff --git a/library/cpp/yt/string/format_string-inl.h b/library/cpp/yt/string/format_string-inl.h index a692d9648d0..67f9bad45c9 100644 --- a/library/cpp/yt/string/format_string-inl.h +++ b/library/cpp/yt/string/format_string-inl.h @@ -4,6 +4,8 @@  #include "format_string.h"  #endif +#include <algorithm> +  namespace NYT {  //////////////////////////////////////////////////////////////////////////////// @@ -15,8 +17,13 @@ consteval TBasicFormatString<TArgs...>::TBasicFormatString(const T& fmt)      : Format_(fmt)  {      CheckFormattability(); -#if !defined(NDEBUG) && !defined(YT_DISABLE_FORMAT_STATIC_ANALYSIS) -    NDetail::TFormatAnalyser::ValidateFormat<std::remove_cvref_t<TArgs>...>(Format_); +#if !defined(YT_DISABLE_FORMAT_STATIC_ANALYSIS) +    std::tie(Markers, Escapes) = NDetail::TFormatAnalyser::AnalyzeFormat<std::remove_cvref_t<TArgs>...>(Format_); +#else +    std::ranges::fill_n(std::ranges::begin(Escapes), 1, -1); +    if constexpr (sizeof...(TArgs) != 0) { +        std::ranges::fill_n(std::ranges::begin(Markers), 1, std::tuple{0, 0}); +    }  #endif  } @@ -46,6 +53,11 @@ template <class... TArgs>  TBasicFormatString<TArgs...>::TBasicFormatString(TRuntimeFormat fmt)      : Format_(fmt.Get())  { +    std::ranges::fill_n(std::ranges::begin(Escapes), 1, -1); +    if constexpr (sizeof...(TArgs) != 0) { +        std::ranges::fill_n(std::ranges::begin(Markers), 1, std::tuple{0, 0}); +    } +      // NB(arkady-e1ppa): StaticFormat performs the      // formattability check of the args in a way      // that provides more useful information diff --git a/library/cpp/yt/string/format_string.h b/library/cpp/yt/string/format_string.h index 786c2e39ed5..1008ccb4533 100644 --- a/library/cpp/yt/string/format_string.h +++ b/library/cpp/yt/string/format_string.h @@ -43,6 +43,10 @@ public:      static consteval void CheckFormattability(); +    // Data used for compile-time slicing of the format string. +    NDetail::TFormatAnalyser::TMarkerLocations<TArgs...> Markers = {}; +    NDetail::TFormatAnalyser::TEscapeLocations Escapes = {}; +  private:      std::string_view Format_; diff --git a/library/cpp/yt/string/unittests/format_ut.cpp b/library/cpp/yt/string/unittests/format_ut.cpp index ac3be99ad97..8aca8c8e290 100644 --- a/library/cpp/yt/string/unittests/format_ut.cpp +++ b/library/cpp/yt/string/unittests/format_ut.cpp @@ -326,6 +326,11 @@ TEST(TFormatTest, CustomFlagsCollectionTwoLevels)      EXPECT_EQ(Format("%NRv", arr), toCollectionD2("RNP"));  } +TEST(TFormatTest, ManyEscapes) +{ +    EXPECT_EQ("a%b%c%d%e%f%g", Format("%v%%%v%%%v%%%v%%%v%%%v%%%g", "a", "b", "c", "d", "e", "f", "g")); +} +  ////////////////////////////////////////////////////////////////////////////////  } // namespace | 
