diff options
author | Maxim Yurchuk <maxim-yurchuk@ydb.tech> | 2024-11-30 22:18:35 +0000 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-11-30 22:18:35 +0000 |
commit | b2a5d3a720980a7304cdf814082ebed47bbf1191 (patch) | |
tree | 7ee20fb54151a1468759da4feee3de61f013ff7d /yt | |
parent | 7466d62733bffe5cb040f37b21c5a9a4ad174353 (diff) | |
parent | b4cba2872f6348770b561394bef1668f54abfe27 (diff) | |
download | ydb-b2a5d3a720980a7304cdf814082ebed47bbf1191.tar.gz |
Merge pull request #12156 from ydb-platform/mergelibs-241130-1815
Library import 241130-1815
Diffstat (limited to 'yt')
40 files changed, 67 insertions, 494 deletions
diff --git a/yt/yt/client/table_client/unittests/ya.make b/yt/yt/client/table_client/unittests/ya.make index f7594f2b9a..a91bc4b66f 100644 --- a/yt/yt/client/table_client/unittests/ya.make +++ b/yt/yt/client/table_client/unittests/ya.make @@ -2,8 +2,6 @@ GTEST(unittester-client-table-client) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -ALLOCATOR(YT) - SRCS( columnar_statistics_ut.cpp columnar_ut.cpp diff --git a/yt/yt/client/table_client/unversioned_row.cpp b/yt/yt/client/table_client/unversioned_row.cpp index 1dce4fb1b4..2c71c617f2 100644 --- a/yt/yt/client/table_client/unversioned_row.cpp +++ b/yt/yt/client/table_client/unversioned_row.cpp @@ -1274,7 +1274,9 @@ void ValidateReadTimestamp(TTimestamp timestamp) timestamp != AsyncLastCommittedTimestamp && (timestamp < MinTimestamp || timestamp > MaxTimestamp)) { - THROW_ERROR_EXCEPTION("Invalid read timestamp %x", timestamp); + THROW_ERROR_EXCEPTION(NTableClient::EErrorCode::TimestampOutOfRange, + "Invalid read timestamp %x", + timestamp); } } diff --git a/yt/yt/client/unittests/ya.make b/yt/yt/client/unittests/ya.make index f5111fca97..a71a98db2c 100644 --- a/yt/yt/client/unittests/ya.make +++ b/yt/yt/client/unittests/ya.make @@ -2,8 +2,6 @@ GTEST(unittester-client) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -ALLOCATOR(YT) - PROTO_NAMESPACE(yt) SRCS( diff --git a/yt/yt/core/actions/unittests/ya.make b/yt/yt/core/actions/unittests/ya.make index b1cf897388..10bbe6d32f 100644 --- a/yt/yt/core/actions/unittests/ya.make +++ b/yt/yt/core/actions/unittests/ya.make @@ -2,10 +2,6 @@ GTEST(unittester-core-actions) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -IF (NOT OS_WINDOWS AND NOT ARCH_AARCH64) - ALLOCATOR(YT) -ENDIF() - PROTO_NAMESPACE(yt) SRCS( diff --git a/yt/yt/core/bus/unittests/ya.make b/yt/yt/core/bus/unittests/ya.make index 7f56a36940..23ff019455 100644 --- a/yt/yt/core/bus/unittests/ya.make +++ b/yt/yt/core/bus/unittests/ya.make @@ -2,10 +2,6 @@ GTEST(unittester-core-bus) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -IF (NOT OS_WINDOWS AND NOT ARCH_AARCH64) - ALLOCATOR(YT) -ENDIF() - PROTO_NAMESPACE(yt) SRCS( diff --git a/yt/yt/core/compression/unittests/ya.make b/yt/yt/core/compression/unittests/ya.make index ec506bdc6f..6f7d0a63a1 100644 --- a/yt/yt/core/compression/unittests/ya.make +++ b/yt/yt/core/compression/unittests/ya.make @@ -2,10 +2,6 @@ GTEST(unittester-core-compression) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -IF (NOT OS_WINDOWS AND NOT ARCH_AARCH64) - ALLOCATOR(YT) -ENDIF() - PROTO_NAMESPACE(yt) SRCS( diff --git a/yt/yt/core/concurrency/fiber_scheduler_thread.cpp b/yt/yt/core/concurrency/fiber_scheduler_thread.cpp index 72130a6717..f4af46f471 100644 --- a/yt/yt/core/concurrency/fiber_scheduler_thread.cpp +++ b/yt/yt/core/concurrency/fiber_scheduler_thread.cpp @@ -20,8 +20,6 @@ #include <library/cpp/yt/global/variable.h> -#include <library/cpp/yt/memory/memory_tag.h> - #include <library/cpp/yt/memory/function_view.h> #include <library/cpp/yt/threading/fork_aware_spin_lock.h> @@ -180,13 +178,6 @@ private: //////////////////////////////////////////////////////////////////////////////// -Y_FORCE_INLINE TMemoryTag SwapMemoryTag(TMemoryTag tag) -{ - auto result = GetCurrentMemoryTag(); - SetCurrentMemoryTag(tag); - return result; -} - Y_FORCE_INLINE TFiberId SwapCurrentFiberId(TFiberId fiberId) { auto result = GetCurrentFiberId(); @@ -818,7 +809,6 @@ protected: void OnSwitch() { FiberId_ = SwapCurrentFiberId(FiberId_); - MemoryTag_ = SwapMemoryTag(MemoryTag_); Fls_ = SwapCurrentFls(Fls_); MinLogLevel_ = SwapMinLogLevel(MinLogLevel_); } @@ -826,13 +816,11 @@ protected: ~TBaseSwitchHandler() { YT_VERIFY(FiberId_ == InvalidFiberId); - YT_VERIFY(MemoryTag_ == NullMemoryTag); YT_VERIFY(!Fls_); YT_VERIFY(MinLogLevel_ == ELogLevel::Minimum); } private: - TMemoryTag MemoryTag_ = NullMemoryTag; TFls* Fls_ = nullptr; TFiberId FiberId_ = InvalidFiberId; ELogLevel MinLogLevel_ = ELogLevel::Minimum; @@ -1147,7 +1135,6 @@ TFiberCanceler GetCurrentFiberCanceler() } if (!switchHandler->Canceler()) { - TMemoryTagGuard guard(NullMemoryTag); switchHandler->Canceler() = New<NDetail::TCanceler>(GetCurrentFiberId()); } @@ -1162,8 +1149,6 @@ void WaitUntilSet(TFuture<void> future, IInvokerPtr invoker) YT_VERIFY(future); YT_ASSERT(invoker); - TMemoryTagGuard memoryTagGuard(NullMemoryTag); - auto* currentFiber = NDetail::TryGetCurrentFiber(); if (!currentFiber) { // When called from a fiber-unfriendly context, we fallback to blocking wait. diff --git a/yt/yt/core/concurrency/fls-inl.h b/yt/yt/core/concurrency/fls-inl.h index 4f8eba6b6f..6267f7f054 100644 --- a/yt/yt/core/concurrency/fls-inl.h +++ b/yt/yt/core/concurrency/fls-inl.h @@ -5,8 +5,6 @@ #endif #undef FLS_INL_H_ -#include <library/cpp/yt/memory/memory_tag.h> - #include <library/cpp/yt/misc/tls.h> namespace NYT::NConcurrency { @@ -97,7 +95,6 @@ Y_FORCE_INLINE T* TFlsSlot<T>::GetOrCreate() const template <class T> T* TFlsSlot<T>::Create() const { - TMemoryTagGuard guard(NullMemoryTag); auto cookie = new T(); GetCurrentFls()->Set(Index_, cookie); return static_cast<T*>(cookie); diff --git a/yt/yt/core/concurrency/nonblocking_batcher.h b/yt/yt/core/concurrency/nonblocking_batcher.h index 5681f27b13..841faf590c 100644 --- a/yt/yt/core/concurrency/nonblocking_batcher.h +++ b/yt/yt/core/concurrency/nonblocking_batcher.h @@ -8,7 +8,7 @@ namespace NYT::NConcurrency { //////////////////////////////////////////////////////////////////////////////// -DEFINE_ENUM(ETNonblockingBatcherTimerState, +DEFINE_ENUM(ENonblockingBatcherTimerState, (Initial) (Started) (Finished) @@ -97,7 +97,7 @@ public: std::vector<TBatch> Drain(); private: - using ETimerState = ETNonblockingBatcherTimerState; + using ETimerState = ENonblockingBatcherTimerState; TBatchLimiter BatchLimiter_; TDuration BatchDuration_; diff --git a/yt/yt/core/concurrency/unittests/scheduler_ut.cpp b/yt/yt/core/concurrency/unittests/scheduler_ut.cpp index 15320fae90..957c1f2ddf 100644 --- a/yt/yt/core/concurrency/unittests/scheduler_ut.cpp +++ b/yt/yt/core/concurrency/unittests/scheduler_ut.cpp @@ -1042,31 +1042,6 @@ TEST_W(TSchedulerTest, CancelDelayedFuture) EXPECT_EQ(NYT::EErrorCode::Generic, error.InnerErrors()[0].GetCode()); } -class TVerifyingMemoryTagGuard -{ -public: - explicit TVerifyingMemoryTagGuard(TMemoryTag tag) - : Tag_(tag) - , SavedTag_(GetCurrentMemoryTag()) - { - SetCurrentMemoryTag(Tag_); - } - - ~TVerifyingMemoryTagGuard() - { - auto tag = GetCurrentMemoryTag(); - EXPECT_EQ(tag, Tag_); - SetCurrentMemoryTag(SavedTag_); - } - - TVerifyingMemoryTagGuard(const TVerifyingMemoryTagGuard& other) = delete; - TVerifyingMemoryTagGuard(TVerifyingMemoryTagGuard&& other) = delete; - -private: - const TMemoryTag Tag_; - const TMemoryTag SavedTag_; -}; - class TWrappingInvoker : public TInvokerWrapper<false> { @@ -1094,44 +1069,6 @@ public: void virtual DoRunCallback(TClosure callback) = 0; }; -class TVerifyingMemoryTaggingInvoker - : public TWrappingInvoker -{ -public: - TVerifyingMemoryTaggingInvoker(IInvokerPtr invoker, TMemoryTag memoryTag) - : TWrappingInvoker(std::move(invoker)) - , MemoryTag_(memoryTag) - { } - -private: - const TMemoryTag MemoryTag_; - - void DoRunCallback(TClosure callback) override - { - TVerifyingMemoryTagGuard memoryTagGuard(MemoryTag_); - callback(); - } -}; - -TEST_W(TSchedulerTest, MemoryTagAndResumer) -{ - auto actionQueue = New<TActionQueue>(); - - auto invoker1 = New<TVerifyingMemoryTaggingInvoker>(actionQueue->GetInvoker(), 1); - auto invoker2 = New<TVerifyingMemoryTaggingInvoker>(actionQueue->GetInvoker(), 2); - - auto asyncResult = BIND([=] { - EXPECT_EQ(GetCurrentMemoryTag(), 1u); - SwitchTo(invoker2); - EXPECT_EQ(GetCurrentMemoryTag(), 1u); - }) - .AsyncVia(invoker1) - .Run(); - - WaitFor(asyncResult) - .ThrowOnError(); -} - void CheckTraceContextTime(const NTracing::TTraceContextPtr& traceContext, TDuration lo, TDuration hi) { auto actual = traceContext->GetElapsedTime(); diff --git a/yt/yt/core/concurrency/unittests/ya.make b/yt/yt/core/concurrency/unittests/ya.make index b8b94dcfc9..61f7b416ad 100644 --- a/yt/yt/core/concurrency/unittests/ya.make +++ b/yt/yt/core/concurrency/unittests/ya.make @@ -2,10 +2,6 @@ GTEST(unittester-core-concurrency) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -IF (NOT OS_WINDOWS AND NOT ARCH_AARCH64) - ALLOCATOR(YT) -ENDIF() - PROTO_NAMESPACE(yt) SRCS( diff --git a/yt/yt/core/crypto/unittests/ya.make b/yt/yt/core/crypto/unittests/ya.make index 460de9e957..d46f2a4729 100644 --- a/yt/yt/core/crypto/unittests/ya.make +++ b/yt/yt/core/crypto/unittests/ya.make @@ -2,10 +2,6 @@ GTEST(unittester-core-crypto) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -IF (NOT OS_WINDOWS AND NOT ARCH_AARCH64) - ALLOCATOR(YT) -ENDIF() - PROTO_NAMESPACE(yt) SRCS( diff --git a/yt/yt/core/http/unittests/ya.make b/yt/yt/core/http/unittests/ya.make index c3724d1234..622590287c 100644 --- a/yt/yt/core/http/unittests/ya.make +++ b/yt/yt/core/http/unittests/ya.make @@ -2,10 +2,6 @@ GTEST(unittester-core-http) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -IF (NOT OS_WINDOWS AND NOT ARCH_AARCH64) - ALLOCATOR(YT) -ENDIF() - PROTO_NAMESPACE(yt) SRCS( diff --git a/yt/yt/core/json/unittests/ya.make b/yt/yt/core/json/unittests/ya.make index 8dc2d207d1..19c3198a5f 100644 --- a/yt/yt/core/json/unittests/ya.make +++ b/yt/yt/core/json/unittests/ya.make @@ -2,10 +2,6 @@ GTEST(unittester-core-json) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -IF (NOT OS_WINDOWS AND NOT ARCH_AARCH64) - ALLOCATOR(YT) -ENDIF() - PROTO_NAMESPACE(yt) SRCS( diff --git a/yt/yt/core/logging/unittests/ya.make b/yt/yt/core/logging/unittests/ya.make index 571cfee61e..3f5d7833de 100644 --- a/yt/yt/core/logging/unittests/ya.make +++ b/yt/yt/core/logging/unittests/ya.make @@ -2,10 +2,6 @@ GTEST(unittester-core-logging) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -IF (NOT OS_WINDOWS AND NOT ARCH_AARCH64) - ALLOCATOR(YT) -ENDIF() - PROTO_NAMESPACE(yt) SRCS( diff --git a/yt/yt/core/misc/error-inl.h b/yt/yt/core/misc/error-inl.h new file mode 100644 index 0000000000..3185de11b2 --- /dev/null +++ b/yt/yt/core/misc/error-inl.h @@ -0,0 +1,24 @@ +#ifndef ERROR_INL_H_ +#error "Direct inclusion of this file is not allowed, include error.h" +// For the sake of sane code completion. +#include "error.h" +#endif + +namespace NYT::NToAttributeValueImpl { + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +NYson::TYsonString TagInvoke(TTagInvokeTag<ToAttributeValue>, const T& value) +{ + return NYson::ConvertToYsonString(value); +} + +inline NYson::TYsonString TagInvoke(TTagInvokeTag<ToAttributeValue>, const NYson::TYsonString& value) +{ + return value; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NToAttributeValueImpl diff --git a/yt/yt/core/misc/error.h b/yt/yt/core/misc/error.h index f3c773da33..8399b49301 100644 --- a/yt/yt/core/misc/error.h +++ b/yt/yt/core/misc/error.h @@ -91,3 +91,7 @@ struct TSerializerTraits< //////////////////////////////////////////////////////////////////////////////// } // namespace NYT + +#define ERROR_INL_H_ +#include "error-inl.h" +#undef ERROR_INL_H_ diff --git a/yt/yt/core/misc/ref_counted_tracker.cpp b/yt/yt/core/misc/ref_counted_tracker.cpp index 85d1e0fd1a..21f9eb815f 100644 --- a/yt/yt/core/misc/ref_counted_tracker.cpp +++ b/yt/yt/core/misc/ref_counted_tracker.cpp @@ -6,8 +6,6 @@ #include <library/cpp/yt/string/format.h> -#include <library/cpp/yt/memory/memory_tag.h> - #include <library/cpp/yt/misc/tls.h> #include <algorithm> @@ -402,8 +400,6 @@ void TRefCountedTracker::FreeSpaceSlow(TRefCountedTypeCookie cookie, size_t spac TRefCountedTracker::TLocalSlot* TRefCountedTracker::GetLocalSlot(TRefCountedTypeCookie cookie) { - TMemoryTagGuard memoryTagGuard(NullMemoryTag); - struct TReclaimer { ~TReclaimer() @@ -460,8 +456,6 @@ TRefCountedTracker::TLocalSlot* TRefCountedTracker::GetLocalSlot(TRefCountedType TRefCountedTracker::TGlobalSlot* TRefCountedTracker::GetGlobalSlot(TRefCountedTypeCookie cookie) { - TMemoryTagGuard memoryTagGuard(NullMemoryTag); - VERIFY_SPINLOCK_AFFINITY(SpinLock_); auto index = cookie.Underlying(); if (index >= std::ssize(GlobalSlots_)) { diff --git a/yt/yt/core/misc/stripped_error.cpp b/yt/yt/core/misc/stripped_error.cpp index e43a61953f..5be93260f9 100644 --- a/yt/yt/core/misc/stripped_error.cpp +++ b/yt/yt/core/misc/stripped_error.cpp @@ -205,7 +205,7 @@ private: // which has minimal API of original dict with backend being the // actual original dict. Once API-related issues are fixed we are // free to implement a backend which doesn't depend on original dict. -std::vector<TString> TErrorAttributes::ListKeys() const +std::vector<TErrorAttributes::TKey> TErrorAttributes::ListKeys() const { auto* attributes = static_cast<IAttributeDictionary*>(Attributes_); if (!attributes) { @@ -223,7 +223,7 @@ std::vector<TErrorAttributes::TKeyValuePair> TErrorAttributes::ListPairs() const return attributes->ListPairs(); } -NYson::TYsonString TErrorAttributes::FindYson(TStringBuf key) const +TErrorAttributes::TValue TErrorAttributes::FindYson(TStringBuf key) const { auto* attributes = static_cast<IAttributeDictionary*>(Attributes_); if (!attributes) { @@ -232,14 +232,14 @@ NYson::TYsonString TErrorAttributes::FindYson(TStringBuf key) const return attributes->FindYson(key); } -void TErrorAttributes::SetYson(const TString& key, const NYson::TYsonString& value) +void TErrorAttributes::SetYson(const TKey& key, const TValue& value) { auto* attributes = static_cast<IAttributeDictionary*>(Attributes_); YT_VERIFY(attributes); return attributes->SetYson(key, value); } -bool TErrorAttributes::Remove(const TString& key) +bool TErrorAttributes::Remove(const TKey& key) { auto* attributes = static_cast<IAttributeDictionary*>(Attributes_); if (!attributes) { @@ -248,7 +248,7 @@ bool TErrorAttributes::Remove(const TString& key) return attributes->Remove(key); } -NYson::TYsonString TErrorAttributes::GetYson(TStringBuf key) const +TErrorAttributes::TValue TErrorAttributes::GetYson(TStringBuf key) const { auto result = FindYson(key); if (!result) { diff --git a/yt/yt/core/misc/stripped_error.h b/yt/yt/core/misc/stripped_error.h index ad985394b8..6dd7b4b87e 100644 --- a/yt/yt/core/misc/stripped_error.h +++ b/yt/yt/core/misc/stripped_error.h @@ -60,25 +60,6 @@ void FormatValue(TStringBuilderBase* builder, TErrorCode code, TStringBuf spec); //////////////////////////////////////////////////////////////////////////////// -struct TErrorAttribute -{ - template <class T> - TErrorAttribute(const TString& key, const T& value) - : Key(key) - , Value(NYson::ConvertToYsonString(value)) - { } - - TErrorAttribute(const TString& key, const NYson::TYsonString& value) - : Key(key) - , Value(value) - { } - - TString Key; - NYson::TYsonString Value; -}; - -//////////////////////////////////////////////////////////////////////////////// - template <class TValue> concept CErrorNestable = requires (TError& error, TValue&& operand) { diff --git a/yt/yt/core/misc/unittests/memory_tag_ut.cpp b/yt/yt/core/misc/unittests/memory_tag_ut.cpp deleted file mode 100644 index f74563f64d..0000000000 --- a/yt/yt/core/misc/unittests/memory_tag_ut.cpp +++ /dev/null @@ -1,243 +0,0 @@ -#include <yt/yt/core/test_framework/framework.h> - -#include <yt/yt/core/actions/invoker_util.h> - -#include <yt/yt/core/concurrency/action_queue.h> -#include <yt/yt/core/concurrency/thread_pool.h> -#include <yt/yt/core/concurrency/scheduler.h> - -#include <library/cpp/yt/memory/memory_tag.h> - -#include <util/random/random.h> - -#include <util/system/compiler.h> - -// These tests do not work under MSAN and ASAN. -#if !defined(_msan_enabled_) and !defined(_asan_enabled_) and defined(_linux_) and defined(YT_ALLOC_ENABLED) - -namespace NYT { -namespace { - -//////////////////////////////////////////////////////////////////////////////// - -// Used for fake side effects to disable compiler optimizations. -volatile const void* FakeSideEffectVolatileVariable = nullptr; - -//////////////////////////////////////////////////////////////////////////////// - -using namespace NConcurrency; -using namespace ::testing; - -//////////////////////////////////////////////////////////////////////////////// - -class TMemoryTagTest - : public TestWithParam<void(*)()> -{ -public: - TMemoryTagTest() = default; -}; - -//////////////////////////////////////////////////////////////////////////////// - -// Allocate vector that results in exactly `size` memory usage considering the 16-byte header. -std::vector<char> MakeAllocation(size_t size) -{ - YT_VERIFY(IsPowerOf2(size)); - - auto result = std::vector<char>(size); - - // We make fake side effect to prevent any compiler optimizations here. - // (Clever compilers like to throw away our unused allocations). - FakeSideEffectVolatileVariable = result.data(); - return result; -} - -//////////////////////////////////////////////////////////////////////////////// - -void TestStackingGuards() -{ - TMemoryTagGuard guard1(1); - EXPECT_EQ(GetMemoryUsageForTag(1), 0u); - auto allocation1 = MakeAllocation(1 << 5); - EXPECT_EQ(GetMemoryUsageForTag(1), 1u << 5); - { - TMemoryTagGuard guard2(2); - auto allocation2 = MakeAllocation(1 << 6); - EXPECT_EQ(GetMemoryUsageForTag(1), 1u << 5); - EXPECT_EQ(GetMemoryUsageForTag(2), 1u << 6); - } - EXPECT_EQ(GetMemoryUsageForTag(1), 1u << 5); - EXPECT_EQ(GetMemoryUsageForTag(2), 0u); - { - TMemoryTagGuard guard2(std::move(guard1)); - auto allocation2 = MakeAllocation(1 << 7); - EXPECT_EQ(GetMemoryUsageForTag(1), (1u << 5) + (1u << 7)); - EXPECT_EQ(GetMemoryUsageForTag(2), 0u); - } - EXPECT_EQ(GetMemoryUsageForTag(1), (1u << 5)); - EXPECT_EQ(GetMemoryUsageForTag(2), 0u); -} - -//////////////////////////////////////////////////////////////////////////////// - -void Action1() -{ - TMemoryTagGuard guard(1); - Yield(); - auto allocation1 = MakeAllocation(1 << 5); - EXPECT_EQ(GetMemoryUsageForTag(1), 1u << 5); - Yield(); - auto allocation2 = MakeAllocation(1 << 7); - EXPECT_EQ(GetMemoryUsageForTag(1), (1u << 5) + (1u << 7)); - Yield(); - auto allocation3 = MakeAllocation(1 << 9); - EXPECT_EQ(GetMemoryUsageForTag(1), (1u << 5) + (1u << 7) + (1u << 9)); -} - -void Action2() -{ - TMemoryTagGuard guard(2); - Yield(); - auto allocation1 = MakeAllocation(1 << 6); - EXPECT_EQ(GetMemoryUsageForTag(2), 1u << 6); - Yield(); - auto allocation2 = MakeAllocation(1 << 8); - EXPECT_EQ(GetMemoryUsageForTag(2), (1u << 6) + (1u << 8)); - Yield(); - auto allocation3 = MakeAllocation(1 << 10); - EXPECT_EQ(GetMemoryUsageForTag(2), (1u << 6) + (1u << 8) + (1u << 10)); -} - -void TestSwitchingFibers() -{ - auto future1 = BIND(&Action1) - .AsyncVia(GetCurrentInvoker()) - .Run(); - auto future2 = BIND(&Action2) - .AsyncVia(GetCurrentInvoker()) - .Run(); - WaitFor(AllSucceeded(std::vector<TFuture<void>>{future1, future2})) - .ThrowOnError(); - EXPECT_EQ(GetMemoryUsageForTag(1), 0u); - EXPECT_EQ(GetMemoryUsageForTag(2), 0u); -} - -//////////////////////////////////////////////////////////////////////////////// - -class TMiniController - : public TRefCounted -{ -public: - TMiniController(IInvokerPtr controlInvoker, TMemoryTag memoryTag) - : MemoryTag_(memoryTag) - , Invoker_(CreateMemoryTaggingInvoker(CreateSerializedInvoker(std::move(controlInvoker)), MemoryTag_)) - { } - - ssize_t GetMemoryUsage() const - { - return GetMemoryUsageForTag(MemoryTag_); - } - - IInvokerPtr GetControlInvoker() const - { - return Invoker_; - } - - std::vector<std::vector<char>>& Allocations() - { - return Allocations_; - } - -private: - TMemoryTag MemoryTag_; - IInvokerPtr Invoker_; - std::vector<std::vector<char>> Allocations_; -}; - -DEFINE_REFCOUNTED_TYPE(TMiniController) -DECLARE_REFCOUNTED_CLASS(TMiniController) - -void Action3(TMiniControllerPtr controller) -{ - controller->Allocations().emplace_back(MakeAllocation(128_MB)); -} - -void TestMemoryTaggingInvoker() -{ - auto queue = New<TActionQueue>(); - auto controller = New<TMiniController>(queue->GetInvoker(), 1); - EXPECT_EQ(controller->GetMemoryUsage(), 0); - - WaitFor(BIND(&Action3, controller) - .AsyncVia(controller->GetControlInvoker()) - .Run()) - .ThrowOnError(); - EXPECT_NEAR(controller->GetMemoryUsage(), 128_MB, 1_MB); - - controller->Allocations().clear(); - controller->Allocations().shrink_to_fit(); - - EXPECT_NEAR(GetMemoryUsageForTag(1), 0, 1_MB); -} - -void TestControllersInThreadPool() -{ - std::vector<TMiniControllerPtr> controllers; - constexpr int controllerCount = 1000; - auto pool = CreateThreadPool(16, "TestPool"); - for (int index = 0; index < controllerCount; ++index) { - controllers.emplace_back(New<TMiniController>(pool->GetInvoker(), index + 1)); - } - constexpr int actionCount = 100 * 1000; - std::vector<TFuture<void>> futures; - std::vector<int> memoryUsages(controllerCount); - srand(42); - for (int index = 0; index < actionCount; ++index) { - int controllerIndex = rand() % controllerCount; - auto allocationSize = 1 << (5 + rand() % 10); - memoryUsages[controllerIndex] += allocationSize; - const auto& controller = controllers[controllerIndex]; - futures.emplace_back( - BIND([] (TMiniControllerPtr controller, int allocationSize) { - controller->Allocations().emplace_back(MakeAllocation(allocationSize)); - }, controller, allocationSize) - .AsyncVia(controller->GetControlInvoker()) - .Run()); - } - WaitFor(AllSucceeded(futures)) - .ThrowOnError(); - for (int index = 0; index < controllerCount; ++index) { - EXPECT_NEAR(memoryUsages[index], controllers[index]->GetMemoryUsage(), 10_KB); - } - controllers.clear(); - for (int index = 0; index < controllerCount; ++index) { - EXPECT_NEAR(GetMemoryUsageForTag(index + 1), 0, 10_KB); - EXPECT_GE(GetMemoryUsageForTag(index + 1), 0u); - } -} - -//////////////////////////////////////////////////////////////////////////////// - -TEST_P(TMemoryTagTest, Test) -{ - // We wrap anything with an outer action queue to make - // fiber-friendly environment. - auto outerQueue = New<TActionQueue>(); - WaitFor(BIND(GetParam()) - .AsyncVia(outerQueue->GetInvoker()) - .Run()) - .ThrowOnError(); -} - -INSTANTIATE_TEST_SUITE_P(MemoryTagTest, TMemoryTagTest, Values( - &TestStackingGuards, - &TestSwitchingFibers, - &TestMemoryTaggingInvoker, - &TestControllersInThreadPool)); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace -} // namespace NYT - -#endif // !defined(_msan_enabled_) diff --git a/yt/yt/core/misc/unittests/ya.make b/yt/yt/core/misc/unittests/ya.make index 3d85710391..a422838270 100644 --- a/yt/yt/core/misc/unittests/ya.make +++ b/yt/yt/core/misc/unittests/ya.make @@ -2,10 +2,6 @@ GTEST(unittester-core-misc) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -IF (NOT OS_WINDOWS AND NOT ARCH_AARCH64) - ALLOCATOR(YT) -ENDIF() - PROTO_NAMESPACE(yt) SRCS( @@ -46,7 +42,6 @@ SRCS( lock_free_hash_table_ut.cpp lru_cache_ut.cpp maybe_inf_ut.cpp - memory_tag_ut.cpp moving_average_ut.cpp mpsc_fair_share_queue_ut.cpp mpsc_stack_ut.cpp diff --git a/yt/yt/core/net/unittests/ya.make b/yt/yt/core/net/unittests/ya.make index d1bf832968..9976ff1c18 100644 --- a/yt/yt/core/net/unittests/ya.make +++ b/yt/yt/core/net/unittests/ya.make @@ -2,10 +2,6 @@ GTEST(unittester-core-net) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -IF (NOT OS_WINDOWS AND NOT ARCH_AARCH64) - ALLOCATOR(YT) -ENDIF() - PROTO_NAMESPACE(yt) SRCS( diff --git a/yt/yt/core/profiling/unittests/ya.make b/yt/yt/core/profiling/unittests/ya.make index b31b812b83..8b9cc6c8de 100644 --- a/yt/yt/core/profiling/unittests/ya.make +++ b/yt/yt/core/profiling/unittests/ya.make @@ -2,10 +2,6 @@ GTEST(unittester-core-profiling) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -IF (NOT OS_WINDOWS AND NOT ARCH_AARCH64) - ALLOCATOR(YT) -ENDIF() - PROTO_NAMESPACE(yt) SRCS( diff --git a/yt/yt/core/rpc/unittests/main/ya.make b/yt/yt/core/rpc/unittests/main/ya.make index 773edcce96..37b494945c 100644 --- a/yt/yt/core/rpc/unittests/main/ya.make +++ b/yt/yt/core/rpc/unittests/main/ya.make @@ -2,10 +2,6 @@ GTEST(unittester-core-rpc) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -IF (NOT OS_WINDOWS AND NOT ARCH_AARCH64) - ALLOCATOR(YT) -ENDIF() - PROTO_NAMESPACE(yt) SRCS( diff --git a/yt/yt/core/rpc/unittests/rpc_allocation_tags_ut.cpp b/yt/yt/core/rpc/unittests/rpc_allocation_tags_ut.cpp index ec5c49d7cc..55dd20bde7 100644 --- a/yt/yt/core/rpc/unittests/rpc_allocation_tags_ut.cpp +++ b/yt/yt/core/rpc/unittests/rpc_allocation_tags_ut.cpp @@ -33,7 +33,7 @@ TYPED_TEST(TRpcTest, ResponseWithAllocationTags) auto previousLimit = memoryUsageTracker->GetLimit(); memoryUsageTracker->SetLimit(2_GB); - static TMemoryTag testMemoryTag = 1 << 20; + static int testMemoryTag = 1 << 20; testMemoryTag++; EnableMemoryProfilingTags(); @@ -70,11 +70,11 @@ TYPED_TEST(TRpcTest, ResponseWithAllocationTags) req2->set_size(size); auto rspFutureProp = req2->Invoke() - .Apply(BIND([testMemoryTag=testMemoryTag] (const TRspPtr& res) { + .Apply(BIND([testMemoryTag = testMemoryTag] (const TRspPtr& res) { auto localContext = TryGetCurrentTraceContext(); EXPECT_NE(localContext, nullptr); if (localContext) { - EXPECT_EQ(localContext->FindAllocationTag<TMemoryTag>(MemoryAllocationTag).value_or(NullMemoryTag), testMemoryTag); + EXPECT_EQ(localContext->FindAllocationTag<int>(MemoryAllocationTag).value_or(NullMemoryTag), testMemoryTag); } return res; }).AsyncVia(actionQueue->GetInvoker())); diff --git a/yt/yt/core/rpc/unittests/rpc_ut.cpp b/yt/yt/core/rpc/unittests/rpc_ut.cpp index d83c1b7d6b..c887795911 100644 --- a/yt/yt/core/rpc/unittests/rpc_ut.cpp +++ b/yt/yt/core/rpc/unittests/rpc_ut.cpp @@ -620,39 +620,6 @@ TYPED_TEST(TNotGrpcTest, Compression) } } -#if !defined(_asan_enabled_) && !defined(_msan_enabled_) && defined(_linux_) - -TYPED_TEST(TRpcTest, ResponseMemoryTag) -{ - static TMemoryTag testMemoryTag = 12345; - testMemoryTag++; - auto initialMemoryUsage = GetMemoryUsageForTag(testMemoryTag); - - std::vector<TTestProxy::TRspPassCallPtr> rsps; - { - TTestProxy proxy(this->CreateChannel()); - TString userName("user"); - - TMemoryTagGuard guard(testMemoryTag); - - for (int i = 0; i < 1000; ++i) { - auto req = proxy.PassCall(); - req->SetUser(userName); - req->SetMutationId(TGuid::Create()); - req->SetRetry(false); - auto err = req->Invoke().Get(); - rsps.push_back(err.ValueOrThrow()); - } - } - - auto currentMemoryUsage = GetMemoryUsageForTag(testMemoryTag); - EXPECT_GE(currentMemoryUsage - initialMemoryUsage, 200_KB) - << "InitialUsage: " << initialMemoryUsage << std::endl - << "Current: " << currentMemoryUsage; -} - -#endif - TYPED_TEST(TNotGrpcTest, RequestBytesThrottling) { auto configText = TString(R"({ diff --git a/yt/yt/core/rpc/unittests/shutdown/ya.make b/yt/yt/core/rpc/unittests/shutdown/ya.make index a55c4d4fc0..ffc34b2bd0 100644 --- a/yt/yt/core/rpc/unittests/shutdown/ya.make +++ b/yt/yt/core/rpc/unittests/shutdown/ya.make @@ -2,10 +2,6 @@ GTEST(unittester-core-rpc-shutdown) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -IF (NOT OS_WINDOWS AND NOT ARCH_AARCH64) - ALLOCATOR(YT) -ENDIF() - PROTO_NAMESPACE(yt) SRCS( diff --git a/yt/yt/core/tracing/unittests/allocation_tags_ut.cpp b/yt/yt/core/tracing/unittests/allocation_tags_ut.cpp index 9c567309ee..86a645ea5f 100644 --- a/yt/yt/core/tracing/unittests/allocation_tags_ut.cpp +++ b/yt/yt/core/tracing/unittests/allocation_tags_ut.cpp @@ -16,7 +16,7 @@ TEST(TAllocationTagsTest, GetSetAllocationTags) ASSERT_EQ(traceContext->FindAllocationTag<std::string>("a"), std::nullopt); traceContext->SetAllocationTags({{"user", "first"}, {"sometag", "my"}}); - ASSERT_EQ(traceContext->FindAllocationTag<TMemoryTag>("memory_tag"), std::nullopt); + ASSERT_EQ(traceContext->FindAllocationTag<int>("memory_tag"), std::nullopt); ASSERT_EQ(traceContext->FindAllocationTag<std::string>("user"), "first"); ASSERT_EQ(traceContext->FindAllocationTag<std::string>("sometag"), "my"); ASSERT_EQ(traceContext->FindAllocationTag<std::string>("other"), std::nullopt); @@ -35,8 +35,8 @@ TEST(TAllocationTagsTest, GetSetAllocationTags) ASSERT_EQ(traceContext->FindAllocationTag<std::string>("sometag"), std::nullopt); ASSERT_TRUE(traceContext->GetAllocationTags().empty()); - traceContext->SetAllocationTag<TMemoryTag>("memory_tag", TMemoryTag{1}); - ASSERT_EQ(traceContext->FindAllocationTag<TMemoryTag>("memory_tag"), TMemoryTag{1}); + traceContext->SetAllocationTag<int>("memory_tag", 1); + ASSERT_EQ(traceContext->FindAllocationTag<int>("memory_tag"), 1); ASSERT_FALSE(traceContext->GetAllocationTags().empty()); } diff --git a/yt/yt/core/ya.make b/yt/yt/core/ya.make index 59362bed77..56334cc6c0 100644 --- a/yt/yt/core/ya.make +++ b/yt/yt/core/ya.make @@ -390,7 +390,7 @@ RECURSE( test_framework ) -IF (NOT OPENSOURCE) +IF (NOT OPENSOURCE AND OS_LINUX) RECURSE( benchmarks bus/benchmarks diff --git a/yt/yt/core/ypath/unittests/ya.make b/yt/yt/core/ypath/unittests/ya.make index fa6d821da4..b4f2240c8c 100644 --- a/yt/yt/core/ypath/unittests/ya.make +++ b/yt/yt/core/ypath/unittests/ya.make @@ -2,10 +2,6 @@ GTEST(unittester-core-ypath) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -IF (NOT OS_WINDOWS AND NOT ARCH_AARCH64) - ALLOCATOR(YT) -ENDIF() - PROTO_NAMESPACE(yt) SRCS( diff --git a/yt/yt/core/yson/unittests/ya.make b/yt/yt/core/yson/unittests/ya.make index a153d12ce7..59a807fca4 100644 --- a/yt/yt/core/yson/unittests/ya.make +++ b/yt/yt/core/yson/unittests/ya.make @@ -2,10 +2,6 @@ GTEST(unittester-core-yson) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -IF (NOT OS_WINDOWS AND NOT ARCH_AARCH64) - ALLOCATOR(YT) -ENDIF() - PROTO_NAMESPACE(yt) SRCS( diff --git a/yt/yt/core/ytree/convert-inl.h b/yt/yt/core/ytree/convert-inl.h index 694aca2d81..0e3385ed5b 100644 --- a/yt/yt/core/ytree/convert-inl.h +++ b/yt/yt/core/ytree/convert-inl.h @@ -182,12 +182,20 @@ T ConstructYTreeConvertibleObject() //////////////////////////////////////////////////////////////////////////////// -namespace { +} // namespace NYT::NYTree + +//////////////////////////////////////////////////////////////////////////////// + +namespace NYT::NConvertToImpl { //////////////////////////////////////////////////////////////////////////////// +namespace { + double ConvertYsonStringBaseToDouble(const NYson::TYsonStringBuf& yson) { + using namespace NYT::NYTree; + NYson::TTokenizer tokenizer(yson.AsStringBuf()); const auto& token = SkipAttributes(&tokenizer); switch (token.GetType()) { @@ -204,8 +212,12 @@ double ConvertYsonStringBaseToDouble(const NYson::TYsonStringBuf& yson) } } +//////////////////////////////////////////////////////////////////////////////// + TString ConvertYsonStringBaseToString(const NYson::TYsonStringBuf& yson) { + using namespace NYT::NYTree; + NYson::TTokenizer tokenizer(yson.AsStringBuf()); const auto& token = SkipAttributes(&tokenizer); switch (token.GetType()) { @@ -218,15 +230,7 @@ TString ConvertYsonStringBaseToString(const NYson::TYsonStringBuf& yson) } } -//////////////////////////////////////////////////////////////////////////////// - -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NYTree - -namespace NYT::NConvertToImpl { +} // namespace //////////////////////////////////////////////////////////////////////////////// @@ -314,25 +318,25 @@ IMPLEMENT_CHECKED_INTEGRAL_CONVERT_TO(ui8) template <> inline double TagInvoke(TTagInvokeTag<ConvertTo<double>>, const NYson::TYsonString& str) { - return NYTree::ConvertYsonStringBaseToDouble(str); + return ConvertYsonStringBaseToDouble(str); } template <> inline double TagInvoke(TTagInvokeTag<ConvertTo<double>>, const NYson::TYsonStringBuf& str) { - return NYTree::ConvertYsonStringBaseToDouble(str); + return ConvertYsonStringBaseToDouble(str); } template <> inline TString TagInvoke(TTagInvokeTag<ConvertTo<TString>>, const NYson::TYsonString& str) { - return NYTree::ConvertYsonStringBaseToString(str); + return ConvertYsonStringBaseToString(str); } template <> inline TString TagInvoke(TTagInvokeTag<ConvertTo<TString>>, const NYson::TYsonStringBuf& str) { - return NYTree::ConvertYsonStringBaseToString(str); + return ConvertYsonStringBaseToString(str); } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/ytree/unittests/ya.make b/yt/yt/core/ytree/unittests/ya.make index 034ba0a1d5..7196cea98c 100644 --- a/yt/yt/core/ytree/unittests/ya.make +++ b/yt/yt/core/ytree/unittests/ya.make @@ -2,10 +2,6 @@ GTEST(unittester-core-ytree) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -IF (NOT OS_WINDOWS AND NOT ARCH_AARCH64) - ALLOCATOR(YT) -ENDIF() - PROTO_NAMESPACE(yt) SRCS( diff --git a/yt/yt/library/auth/unittests/ya.make b/yt/yt/library/auth/unittests/ya.make index c45504d10a..e72e513258 100644 --- a/yt/yt/library/auth/unittests/ya.make +++ b/yt/yt/library/auth/unittests/ya.make @@ -2,8 +2,6 @@ GTEST(unittester-library-auth) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -ALLOCATOR(YT) - SRCS( auth_ut.cpp ) diff --git a/yt/yt/library/decimal/unittests/ya.make b/yt/yt/library/decimal/unittests/ya.make index 76341ee7b2..172760632d 100644 --- a/yt/yt/library/decimal/unittests/ya.make +++ b/yt/yt/library/decimal/unittests/ya.make @@ -2,8 +2,6 @@ GTEST(unittester-library-decimal) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -ALLOCATOR(YT) - SRCS( decimal_ut.cpp ) diff --git a/yt/yt/library/erasure/impl/unittests/ya.make b/yt/yt/library/erasure/impl/unittests/ya.make index 15b3f6fb8d..d2b30ea140 100644 --- a/yt/yt/library/erasure/impl/unittests/ya.make +++ b/yt/yt/library/erasure/impl/unittests/ya.make @@ -2,8 +2,6 @@ GTEST(unittester-library-erasure) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -ALLOCATOR(YT) - SRCS( erasure_stability_ut.cpp ) diff --git a/yt/yt/library/process/unittests/ya.make b/yt/yt/library/process/unittests/ya.make index 7c5d0cb48f..149d9eee1f 100644 --- a/yt/yt/library/process/unittests/ya.make +++ b/yt/yt/library/process/unittests/ya.make @@ -2,8 +2,6 @@ GTEST(unittester-library-process) INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) -ALLOCATOR(YT) - SRCS( pipes_ut.cpp process_ut.cpp diff --git a/yt/yt/library/tvm/service/unittests/ya.make b/yt/yt/library/tvm/service/unittests/ya.make index 28629e6d82..23ac522bd0 100644 --- a/yt/yt/library/tvm/service/unittests/ya.make +++ b/yt/yt/library/tvm/service/unittests/ya.make @@ -1,7 +1,5 @@ GTEST(unittester-library-auth_tvm) -ALLOCATOR(YT) - INCLUDE(${ARCADIA_ROOT}/yt/opensource.inc) PEERDIR( diff --git a/yt/yt/library/ytprof/unittests/heap_profiler_ut.cpp b/yt/yt/library/ytprof/unittests/heap_profiler_ut.cpp index 6935216130..a7a19b8eb9 100644 --- a/yt/yt/library/ytprof/unittests/heap_profiler_ut.cpp +++ b/yt/yt/library/ytprof/unittests/heap_profiler_ut.cpp @@ -64,9 +64,9 @@ TEST(THeapProfilerTest, ReadProfile) auto h0 = BlowHeap<0>(); - auto tag = TMemoryTag(1); + int tag = 1; traceContext->SetAllocationTags({{"user", "second"}, {"sometag", "notmy"}, {MemoryAllocationTagKey, ToString(tag)}}); - auto currentTag = traceContext->FindAllocationTag<TMemoryTag>(MemoryAllocationTagKey); + auto currentTag = traceContext->FindAllocationTag<int>(MemoryAllocationTagKey); ASSERT_EQ(currentTag, tag); auto h1 = BlowHeap<1>(); @@ -99,7 +99,7 @@ TEST(THeapProfilerTest, ReadProfile) output.Finish(); } -TEST(THeapProfilerTest, AllocationTagsWithMemoryTag) +TEST(THeapProfilerTest, AllocationTags) { EnableMemoryProfilingTags(); auto traceContext = TTraceContext::NewRoot("Root"); @@ -201,7 +201,7 @@ TEST(THeapProfilerTest, HugeAllocationsTagsWithMemoryTag) heap.push_back(BlowHeap<0>()); traceContext->SetAllocationTag(MemoryAllocationTagKey, MemoryAllocationTagValues[1]); - ASSERT_EQ(traceContext->FindAllocationTag<TMemoryTag>(MemoryAllocationTagKey), 1); + ASSERT_EQ(traceContext->FindAllocationTag<int>(MemoryAllocationTagKey), 1); heap.push_back(BlowHeap<1>(100)); |