diff options
author | lukyan <lukyan@yandex-team.ru> | 2022-02-10 16:48:13 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:48:13 +0300 |
commit | 3e359c7e6344b01b8d0b0fc619297ffdc2644c49 (patch) | |
tree | 84fef6ce6ee76479e37582b357af732385cffdcf | |
parent | a02ea31d51f94b7edb98c6ab1322bf26ca04beb5 (diff) | |
download | ydb-3e359c7e6344b01b8d0b0fc619297ffdc2644c49.tar.gz |
Restoring authorship annotation for <lukyan@yandex-team.ru>. Commit 1 of 2.
30 files changed, 1429 insertions, 1429 deletions
diff --git a/contrib/libs/cxxsupp/libcxx/include/__wrappers_config b/contrib/libs/cxxsupp/libcxx/include/__wrappers_config index 934e7c9279..60199c0093 100644 --- a/contrib/libs/cxxsupp/libcxx/include/__wrappers_config +++ b/contrib/libs/cxxsupp/libcxx/include/__wrappers_config @@ -2,12 +2,12 @@ #if !defined(NDEBUG) && !defined(_LIBCPP_DEBUG) # define _LIBCPP_DEBUG 0 -#endif +#endif #ifndef _YNDX_LIBCPP_ENABLE_EXTENSIONS # define _YNDX_LIBCPP_ENABLE_EXTENSIONS 1 -#endif - +#endif + #if _YNDX_LIBCPP_ENABLE_EXTENSIONS == 1 # define _YNDX_LIBCXX_ENABLE_STRING_RESIZE_UNINITIALIZED 1 # define _YNDX_LIBCXX_ENABLE_VECTOR_BOOL_COMPRESSION 0 diff --git a/library/cpp/linear_regression/linear_regression.cpp b/library/cpp/linear_regression/linear_regression.cpp index 150f9d214e..bcab777133 100644 --- a/library/cpp/linear_regression/linear_regression.cpp +++ b/library/cpp/linear_regression/linear_regression.cpp @@ -33,7 +33,7 @@ bool TFastLinearRegressionSolver::Add(const TVector<double>& features, const dou AddFeaturesProduct(weight, features, LinearizedOLSMatrix); const double weightedGoal = goal * weight; - double* olsVectorElement = OLSVector.data(); + double* olsVectorElement = OLSVector.data(); for (const double feature : features) { *olsVectorElement += feature * weightedGoal; ++olsVectorElement; @@ -72,31 +72,31 @@ bool TLinearRegressionSolver::Add(const TVector<double>& features, const double ; } - double* olsMatrixElement = LinearizedOLSMatrix.data(); + double* olsMatrixElement = LinearizedOLSMatrix.data(); - const double* lastMean = LastMeans.data(); - const double* newMean = NewMeans.data(); - const double* lastMeansEnd = lastMean + LastMeans.size(); - const double* newMeansEnd = newMean + NewMeans.size(); + const double* lastMean = LastMeans.data(); + const double* newMean = NewMeans.data(); + const double* lastMeansEnd = lastMean + LastMeans.size(); + const double* newMeansEnd = newMean + NewMeans.size(); #ifdef _sse2_ - for (; lastMean != lastMeansEnd; ++lastMean, ++newMean) { + for (; lastMean != lastMeansEnd; ++lastMean, ++newMean) { __m128d factor = _mm_set_pd(*lastMean, *lastMean); const double* secondFeatureMean = newMean; - for (; secondFeatureMean + 1 < newMeansEnd; secondFeatureMean += 2, olsMatrixElement += 2) { + for (; secondFeatureMean + 1 < newMeansEnd; secondFeatureMean += 2, olsMatrixElement += 2) { __m128d matrixElem = _mm_loadu_pd(olsMatrixElement); __m128d secondFeatureMeanElem = _mm_loadu_pd(secondFeatureMean); __m128d product = _mm_mul_pd(factor, secondFeatureMeanElem); __m128d addition = _mm_add_pd(matrixElem, product); _mm_storeu_pd(olsMatrixElement, addition); } - for (; secondFeatureMean < newMeansEnd; ++secondFeatureMean) { + for (; secondFeatureMean < newMeansEnd; ++secondFeatureMean) { *olsMatrixElement++ += *lastMean * *secondFeatureMean; } } #else - for (; lastMean != lastMeansEnd; ++lastMean, ++newMean) { - for (const double* secondFeatureMean = newMean; secondFeatureMean < newMeansEnd; ++secondFeatureMean) { + for (; lastMean != lastMeansEnd; ++lastMean, ++newMean) { + for (const double* secondFeatureMean = newMean; secondFeatureMean < newMeansEnd; ++secondFeatureMean) { *olsMatrixElement++ += *lastMean * *secondFeatureMean; } } @@ -326,13 +326,13 @@ namespace { #ifdef _sse2_ inline void AddFeaturesProduct(const double weight, const TVector<double>& features, TVector<double>& linearizedOLSTriangleMatrix) { - const double* leftFeature = features.data(); - const double* featuresEnd = features.data() + features.size(); - double* matrixElement = linearizedOLSTriangleMatrix.data(); + const double* leftFeature = features.data(); + const double* featuresEnd = features.data() + features.size(); + double* matrixElement = linearizedOLSTriangleMatrix.data(); size_t unaligned = features.size() & 0x1; - for (; leftFeature != featuresEnd; ++leftFeature, ++matrixElement) { + for (; leftFeature != featuresEnd; ++leftFeature, ++matrixElement) { const double weightedFeature = weight * *leftFeature; const double* rightFeature = leftFeature; __m128d wf = {weightedFeature, weightedFeature}; @@ -340,7 +340,7 @@ namespace { *matrixElement += weightedFeature * *rightFeature; } unaligned = (unaligned + 1) & 0x1; - for (; rightFeature != featuresEnd; rightFeature += 2, matrixElement += 2) { + for (; rightFeature != featuresEnd; rightFeature += 2, matrixElement += 2) { __m128d rf = _mm_loadu_pd(rightFeature); __m128d matrixRow = _mm_loadu_pd(matrixElement); __m128d rowAdd = _mm_mul_pd(rf, wf); @@ -352,13 +352,13 @@ namespace { } #else inline void AddFeaturesProduct(const double weight, const TVector<double>& features, TVector<double>& linearizedTriangleMatrix) { - const double* leftFeature = features.data(); - const double* featuresEnd = features.data() + features.size(); - double* matrixElement = linearizedTriangleMatrix.data(); - for (; leftFeature != featuresEnd; ++leftFeature, ++matrixElement) { + const double* leftFeature = features.data(); + const double* featuresEnd = features.data() + features.size(); + double* matrixElement = linearizedTriangleMatrix.data(); + for (; leftFeature != featuresEnd; ++leftFeature, ++matrixElement) { const double weightedFeature = weight * *leftFeature; const double* rightFeature = leftFeature; - for (; rightFeature != featuresEnd; ++rightFeature, ++matrixElement) { + for (; rightFeature != featuresEnd; ++rightFeature, ++matrixElement) { *matrixElement += weightedFeature * *rightFeature; } *matrixElement += weightedFeature; diff --git a/library/cpp/linear_regression/linear_regression.h b/library/cpp/linear_regression/linear_regression.h index e57de5ff6c..5015e2e4b6 100644 --- a/library/cpp/linear_regression/linear_regression.h +++ b/library/cpp/linear_regression/linear_regression.h @@ -147,12 +147,12 @@ public: bool Add(const TVector<double>& features, const TVector<double>& goals) { Y_ASSERT(features.size() == goals.size()); - return Add(features.data(), features.data() + features.size(), goals.data()); + return Add(features.data(), features.data() + features.size(), goals.data()); } bool Add(const TVector<double>& features, const TVector<double>& goals, const TVector<double>& weights) { Y_ASSERT(features.size() == goals.size() && features.size() == weights.size()); - return Add(features.data(), features.data() + features.size(), goals.data(), weights.data()); + return Add(features.data(), features.data() + features.size(), goals.data(), weights.data()); } template <typename TFloatType> diff --git a/library/cpp/testing/benchmark/bench.cpp b/library/cpp/testing/benchmark/bench.cpp index 08d8708005..8d69d64080 100644 --- a/library/cpp/testing/benchmark/bench.cpp +++ b/library/cpp/testing/benchmark/bench.cpp @@ -591,7 +591,7 @@ int NBench::Main(int argc, char** argv) { if (opts.Threads > 1) { NYmp::SetThreadCount(opts.Threads); - NYmp::ParallelForStaticChunk(tests.data(), tests.data() + tests.size(), 1, func); + NYmp::ParallelForStaticChunk(tests.data(), tests.data() + tests.size(), 1, func); } else { for (auto it : tests) { func(&it); diff --git a/library/cpp/yt/coding/varint-inl.h b/library/cpp/yt/coding/varint-inl.h index f0a09e9d30..c226d7de22 100644 --- a/library/cpp/yt/coding/varint-inl.h +++ b/library/cpp/yt/coding/varint-inl.h @@ -31,14 +31,14 @@ Y_FORCE_INLINE int WriteVarUint64Impl(TWriteCallback doWrite, ui64 value) // These are optimized versions of these Read/Write functions in protobuf/io/coded_stream.cc. Y_FORCE_INLINE int WriteVarUint64(IOutputStream* output, ui64 value) { - return WriteVarUint64Impl([&] (ui8 byte) { + return WriteVarUint64Impl([&] (ui8 byte) { output->Write(byte); }, value); } Y_FORCE_INLINE int WriteVarUint64(char* output, ui64 value) { - return WriteVarUint64Impl([&] (ui8 byte) { + return WriteVarUint64Impl([&] (ui8 byte) { *output++ = byte; }, value); } @@ -48,17 +48,17 @@ Y_FORCE_INLINE int WriteVarUint64(char* output, ui64 value) template <class TOutput> Y_FORCE_INLINE int WriteVarUint32Impl(TOutput output, ui32 value) { - return WriteVarUint64(output, static_cast<ui64>(value)); + return WriteVarUint64(output, static_cast<ui64>(value)); } Y_FORCE_INLINE int WriteVarUint32(IOutputStream* output, ui32 value) { - return WriteVarUint32Impl(output, value); + return WriteVarUint32Impl(output, value); } Y_FORCE_INLINE int WriteVarUint32(char* output, ui32 value) { - return WriteVarUint32Impl(output, value); + return WriteVarUint32Impl(output, value); } //////////////////////////////////////////////////////////////////////////////// @@ -66,7 +66,7 @@ Y_FORCE_INLINE int WriteVarUint32(char* output, ui32 value) template <class TOutput> Y_FORCE_INLINE int WriteVarInt32Impl(TOutput output, i32 value) { - return WriteVarUint64(output, static_cast<ui64>(ZigZagEncode32(value))); + return WriteVarUint64(output, static_cast<ui64>(ZigZagEncode32(value))); } Y_FORCE_INLINE int WriteVarInt32(IOutputStream* output, i32 value) @@ -84,7 +84,7 @@ Y_FORCE_INLINE int WriteVarInt32(char* output, i32 value) template <class TOutput> Y_FORCE_INLINE int WriteVarInt64Impl(TOutput output, i64 value) { - return WriteVarUint64(output, static_cast<ui64>(ZigZagEncode64(value))); + return WriteVarUint64(output, static_cast<ui64>(ZigZagEncode64(value))); } Y_FORCE_INLINE int WriteVarInt64(IOutputStream* output, i64 value) @@ -121,7 +121,7 @@ Y_FORCE_INLINE int ReadVarUint64Impl(TReadCallback doRead, ui64* value) Y_FORCE_INLINE int ReadVarUint64(IInputStream* input, ui64* value) { - return ReadVarUint64Impl([&] () { + return ReadVarUint64Impl([&] () { char byte; if (input->Read(&byte, 1) != 1) { throw TSimpleException("Premature end of stream while reading varuint64"); @@ -132,7 +132,7 @@ Y_FORCE_INLINE int ReadVarUint64(IInputStream* input, ui64* value) Y_FORCE_INLINE int ReadVarUint64(const char* input, ui64* value) { - return ReadVarUint64Impl([&] () { + return ReadVarUint64Impl([&] () { char byte = *input; ++input; return byte; diff --git a/library/cpp/yt/coding/varint.h b/library/cpp/yt/coding/varint.h index c5399f8b06..e79db873ae 100644 --- a/library/cpp/yt/coding/varint.h +++ b/library/cpp/yt/coding/varint.h @@ -16,7 +16,7 @@ constexpr size_t MaxVarUint64Size = (8 * sizeof(ui64) - 1) / 7 + 1; constexpr size_t MaxVarInt32Size = (8 * sizeof(ui32) - 1) / 7 + 1; constexpr size_t MaxVarUint32Size = (8 * sizeof(ui32) - 1) / 7 + 1; - + // Various functions to read/write varints. // Returns the number of bytes written. @@ -25,8 +25,8 @@ int WriteVarUint32(IOutputStream* output, ui32 value); int WriteVarInt32(IOutputStream* output, i32 value); int WriteVarInt64(IOutputStream* output, i64 value); -int WriteVarUint64(char* output, ui64 value); -int WriteVarUint32(char* output, ui32 value); +int WriteVarUint64(char* output, ui64 value); +int WriteVarUint32(char* output, ui32 value); int WriteVarInt32(char* output, i32 value); int WriteVarInt64(char* output, i64 value); @@ -36,8 +36,8 @@ int ReadVarUint32(IInputStream* input, ui32* value); int ReadVarInt32(IInputStream* input, i32* value); int ReadVarInt64(IInputStream* input, i64* value); -int ReadVarUint64(const char* input, ui64* value); -int ReadVarUint32(const char* input, ui32* value); +int ReadVarUint64(const char* input, ui64* value); +int ReadVarUint32(const char* input, ui32* value); int ReadVarInt32(const char* input, i32* value); int ReadVarInt64(const char* input, i64* value); diff --git a/library/cpp/yt/memory/blob.cpp b/library/cpp/yt/memory/blob.cpp index 86000b033b..8dfaeb791a 100644 --- a/library/cpp/yt/memory/blob.cpp +++ b/library/cpp/yt/memory/blob.cpp @@ -1,6 +1,6 @@ -#include "blob.h" -#include "ref.h" - +#include "blob.h" +#include "ref.h" + #include <library/cpp/ytalloc/api/ytalloc.h> namespace NYT { @@ -9,26 +9,26 @@ namespace NYT { static constexpr size_t InitialBlobCapacity = 16; static constexpr double BlobCapacityMultiplier = 1.5; - + TBlob::TBlob( TRefCountedTypeCookie tagCookie, size_t size, bool initiailizeStorage, bool pageAligned) : PageAligned_(pageAligned) -{ +{ SetTagCookie(tagCookie); - if (size == 0) { - Reset(); - } else { + if (size == 0) { + Reset(); + } else { Allocate(std::max(size, InitialBlobCapacity)); Size_ = size; - if (initiailizeStorage) { + if (initiailizeStorage) { ::memset(Begin_, 0, Size_); - } - } -} - + } + } +} + TBlob::TBlob( TRefCountedTypeCookie tagCookie, TRef data, @@ -40,80 +40,80 @@ TBlob::TBlob( Append(data); } -TBlob::TBlob(const TBlob& other) +TBlob::TBlob(const TBlob& other) : PageAligned_(other.PageAligned_) -{ +{ SetTagCookie(other); - if (other.Size_ == 0) { - Reset(); - } else { + if (other.Size_ == 0) { + Reset(); + } else { Allocate(std::max(InitialBlobCapacity, other.Size_)); ::memcpy(Begin_, other.Begin_, other.Size_); Size_ = other.Size_; - } -} - + } +} + TBlob::TBlob(TBlob&& other) noexcept : Begin_(other.Begin_) - , Size_(other.Size_) - , Capacity_(other.Capacity_) + , Size_(other.Size_) + , Capacity_(other.Capacity_) , PageAligned_(other.PageAligned_) -{ +{ SetTagCookie(other); - other.Reset(); -} - -TBlob::~TBlob() -{ + other.Reset(); +} + +TBlob::~TBlob() +{ Free(); -} - -void TBlob::Reserve(size_t newCapacity) -{ - if (newCapacity > Capacity_) { +} + +void TBlob::Reserve(size_t newCapacity) +{ + if (newCapacity > Capacity_) { Reallocate(newCapacity); - } -} - -void TBlob::Resize(size_t newSize, bool initializeStorage /*= true*/) -{ - if (newSize > Size_) { - if (newSize > Capacity_) { + } +} + +void TBlob::Resize(size_t newSize, bool initializeStorage /*= true*/) +{ + if (newSize > Size_) { + if (newSize > Capacity_) { size_t newCapacity; - if (Capacity_ == 0) { + if (Capacity_ == 0) { newCapacity = std::max(InitialBlobCapacity, newSize); - } else { + } else { newCapacity = std::max(static_cast<size_t>(Capacity_ * BlobCapacityMultiplier), newSize); - } + } Reallocate(newCapacity); - } - if (initializeStorage) { + } + if (initializeStorage) { ::memset(Begin_ + Size_, 0, newSize - Size_); - } - } - Size_ = newSize; -} - -TBlob& TBlob::operator = (const TBlob& rhs) -{ - if (this != &rhs) { + } + } + Size_ = newSize; +} + +TBlob& TBlob::operator = (const TBlob& rhs) +{ + if (this != &rhs) { this->~TBlob(); new(this) TBlob(rhs); - } - return *this; -} - + } + return *this; +} + TBlob& TBlob::operator = (TBlob&& rhs) noexcept -{ - if (this != &rhs) { +{ + if (this != &rhs) { this->~TBlob(); new(this) TBlob(std::move(rhs)); - } - return *this; -} - -void TBlob::Append(const void* data, size_t size) -{ + } + return *this; +} + +void TBlob::Append(const void* data, size_t size) +{ if (Size_ + size > Capacity_) { Resize(Size_ + size, false); ::memcpy(Begin_ + Size_ - size, data, size); @@ -121,13 +121,13 @@ void TBlob::Append(const void* data, size_t size) ::memcpy(Begin_ + Size_, data, size); Size_ += size; } -} - +} + void TBlob::Append(TRef ref) -{ - Append(ref.Begin(), ref.Size()); -} - +{ + Append(ref.Begin(), ref.Size()); +} + void TBlob::Append(char ch) { if (Size_ + 1 > Capacity_) { @@ -138,12 +138,12 @@ void TBlob::Append(char ch) } } -void TBlob::Reset() -{ +void TBlob::Reset() +{ Begin_ = nullptr; - Size_ = Capacity_ = 0; -} - + Size_ = Capacity_ = 0; +} + char* TBlob::DoAllocate(size_t size) { return static_cast<char*>(PageAligned_ @@ -219,6 +219,6 @@ void swap(TBlob& left, TBlob& right) } } -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yt/memory/blob.h b/library/cpp/yt/memory/blob.h index 99441fb8c9..7d1621a09a 100644 --- a/library/cpp/yt/memory/blob.h +++ b/library/cpp/yt/memory/blob.h @@ -1,25 +1,25 @@ -#pragma once - +#pragma once + #include "ref.h" #include "ref_counted.h" - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + //! Default memory tag for TBlob. struct TDefaultBlobTag { }; -//! A home-grown optimized replacement for |std::vector<char>| suitable for carrying -//! large chunks of data. -/*! - * Compared to |std::vector<char>|, this class supports uninitialized allocations - * when explicitly requested to. - */ -class TBlob -{ -public: +//! A home-grown optimized replacement for |std::vector<char>| suitable for carrying +//! large chunks of data. +/*! + * Compared to |std::vector<char>|, this class supports uninitialized allocations + * when explicitly requested to. + */ +class TBlob +{ +public: //! Constructs a blob with a given size. TBlob( TRefCountedTypeCookie tagCookie, @@ -33,13 +33,13 @@ public: TRef data, bool pageAligned = false); - //! Constructs an empty blob. + //! Constructs an empty blob. template <class TTag = TDefaultBlobTag> explicit TBlob(TTag tag = {}) : TBlob(tag, 0, true, false) { } - - //! Constructs a blob with a given size. + + //! Constructs a blob with a given size. template <class TTag> explicit TBlob( TTag, @@ -52,7 +52,7 @@ public: initiailizeStorage, pageAligned) { } - + //! Copies a chunk of memory into a new instance. template <class TTag> TBlob( @@ -73,52 +73,52 @@ public: template <typename T, typename U> TBlob(const T*, U) = delete; - //! Copies the data. - TBlob(const TBlob& other); - - //! Moves the data (takes the ownership). + //! Copies the data. + TBlob(const TBlob& other); + + //! Moves the data (takes the ownership). TBlob(TBlob&& other) noexcept; - - //! Reclaims the memory. - ~TBlob(); - - //! Ensures that capacity is at least #capacity. - void Reserve(size_t newCapacity); - - //! Changes the size to #newSize. - /*! - * If #size exceeds the current capacity, - * we make sure the new capacity grows exponentially. - * Hence calling #Resize N times to increase the size by N only - * takes amortized O(1) time per call. - */ - void Resize(size_t newSize, bool initializeStorage = true); - - //! Returns the start pointer. + + //! Reclaims the memory. + ~TBlob(); + + //! Ensures that capacity is at least #capacity. + void Reserve(size_t newCapacity); + + //! Changes the size to #newSize. + /*! + * If #size exceeds the current capacity, + * we make sure the new capacity grows exponentially. + * Hence calling #Resize N times to increase the size by N only + * takes amortized O(1) time per call. + */ + void Resize(size_t newSize, bool initializeStorage = true); + + //! Returns the start pointer. Y_FORCE_INLINE const char* Begin() const - { - return Begin_; - } - - //! Returns the start pointer. + { + return Begin_; + } + + //! Returns the start pointer. Y_FORCE_INLINE char* Begin() - { - return Begin_; - } - - //! Returns the end pointer. + { + return Begin_; + } + + //! Returns the end pointer. Y_FORCE_INLINE const char* End() const - { - return Begin_ + Size_; - } - - //! Returns the end pointer. + { + return Begin_ + Size_; + } + + //! Returns the end pointer. Y_FORCE_INLINE char* End() - { - return Begin_ + Size_; - } - - //! Returns the size. + { + return Begin_ + Size_; + } + + //! Returns the size. Y_FORCE_INLINE size_t size() const { return Size_; @@ -126,16 +126,16 @@ public: //! Returns the size. Y_FORCE_INLINE size_t Size() const - { - return Size_; - } - - //! Returns the capacity. + { + return Size_; + } + + //! Returns the capacity. Y_FORCE_INLINE size_t Capacity() const - { - return Capacity_; - } - + { + return Capacity_; + } + //! Returns the TStringBuf instance for the occupied part of the blob. Y_FORCE_INLINE TStringBuf ToStringBuf() const { @@ -148,42 +148,42 @@ public: return TRef(Begin_, Size_); } - //! Provides by-value access to the underlying storage. + //! Provides by-value access to the underlying storage. Y_FORCE_INLINE char operator [] (size_t index) const - { - return Begin_[index]; - } - - //! Provides by-ref access to the underlying storage. + { + return Begin_[index]; + } + + //! Provides by-ref access to the underlying storage. Y_FORCE_INLINE char& operator [] (size_t index) - { - return Begin_[index]; - } - - //! Clears the instance but does not reclaim the memory. + { + return Begin_[index]; + } + + //! Clears the instance but does not reclaim the memory. Y_FORCE_INLINE void Clear() - { - Size_ = 0; - } - - //! Returns |true| if size is zero. + { + Size_ = 0; + } + + //! Returns |true| if size is zero. Y_FORCE_INLINE bool IsEmpty() const - { - return Size_ == 0; - } - - //! Overwrites the current instance. - TBlob& operator = (const TBlob& rhs); - - //! Takes the ownership. + { + return Size_ == 0; + } + + //! Overwrites the current instance. + TBlob& operator = (const TBlob& rhs); + + //! Takes the ownership. TBlob& operator = (TBlob&& rhs) noexcept; - - //! Appends a chunk of memory to the end. - void Append(const void* data, size_t size); - - //! Appends a chunk of memory to the end. + + //! Appends a chunk of memory to the end. + void Append(const void* data, size_t size); + + //! Appends a chunk of memory to the end. void Append(TRef ref); - + //! Appends a single char to the end. void Append(char ch); @@ -192,12 +192,12 @@ public: friend void swap(TBlob& left, TBlob& right); -private: +private: char* Begin_ = nullptr; size_t Size_ = 0; size_t Capacity_ = 0; bool PageAligned_ = false; - + #ifdef YT_ENABLE_REF_COUNTED_TRACKING TRefCountedTypeCookie TagCookie_ = NullRefCountedTypeCookie; #endif @@ -207,15 +207,15 @@ private: void Reallocate(size_t newCapacity); void Free(); - void Reset(); - + void Reset(); + void SetTagCookie(TRefCountedTypeCookie tagCookie); void SetTagCookie(const TBlob& other); -}; - +}; + void swap(TBlob& left, TBlob& right); -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT - +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT + diff --git a/library/cpp/yt/memory/intrusive_ptr.h b/library/cpp/yt/memory/intrusive_ptr.h index 3dead7db1d..7dd02c9c76 100644 --- a/library/cpp/yt/memory/intrusive_ptr.h +++ b/library/cpp/yt/memory/intrusive_ptr.h @@ -1,6 +1,6 @@ #pragma once -#include "ref_counted.h" +#include "ref_counted.h" #include <util/generic/hash.h> #include <util/generic/utility.h> @@ -12,13 +12,13 @@ namespace NYT { //////////////////////////////////////////////////////////////////////////////// -template <class T> +template <class T> class TIntrusivePtr { public: typedef T TUnderlying; - constexpr TIntrusivePtr() noexcept + constexpr TIntrusivePtr() noexcept { } constexpr TIntrusivePtr(std::nullptr_t) noexcept @@ -32,7 +32,7 @@ public: * Note that it notoriously hard to make this constructor explicit * given the current amount of code written. */ - TIntrusivePtr(T* obj, bool addReference = true) noexcept + TIntrusivePtr(T* obj, bool addReference = true) noexcept : T_(obj) { if (T_ && addReference) { @@ -41,7 +41,7 @@ public: } //! Copy constructor. - TIntrusivePtr(const TIntrusivePtr& other) noexcept + TIntrusivePtr(const TIntrusivePtr& other) noexcept : T_(other.Get()) { if (T_) { @@ -51,19 +51,19 @@ public: //! Copy constructor with an upcast. template <class U, class = typename std::enable_if_t<std::is_convertible_v<U*, T*>>> - TIntrusivePtr(const TIntrusivePtr<U>& other) noexcept + TIntrusivePtr(const TIntrusivePtr<U>& other) noexcept : T_(other.Get()) { - static_assert( - std::is_base_of_v<TRefCountedBase, T>, - "Cast allowed only for types derived from TRefCountedBase"); + static_assert( + std::is_base_of_v<TRefCountedBase, T>, + "Cast allowed only for types derived from TRefCountedBase"); if (T_) { Ref(T_); } } //! Move constructor. - TIntrusivePtr(TIntrusivePtr&& other) noexcept + TIntrusivePtr(TIntrusivePtr&& other) noexcept : T_(other.Get()) { other.T_ = nullptr; @@ -71,12 +71,12 @@ public: //! Move constructor with an upcast. template <class U, class = typename std::enable_if_t<std::is_convertible_v<U*, T*>>> - TIntrusivePtr(TIntrusivePtr<U>&& other) noexcept + TIntrusivePtr(TIntrusivePtr<U>&& other) noexcept : T_(other.Get()) { - static_assert( - std::is_base_of_v<TRefCountedBase, T>, - "Cast allowed only for types derived from TRefCountedBase"); + static_assert( + std::is_base_of_v<TRefCountedBase, T>, + "Cast allowed only for types derived from TRefCountedBase"); other.T_ = nullptr; } @@ -101,10 +101,10 @@ public: { static_assert( std::is_convertible_v<U*, T*>, - "U* must be convertible to T*"); - static_assert( - std::is_base_of_v<TRefCountedBase, T>, - "Cast allowed only for types derived from TRefCountedBase"); + "U* must be convertible to T*"); + static_assert( + std::is_base_of_v<TRefCountedBase, T>, + "Cast allowed only for types derived from TRefCountedBase"); TIntrusivePtr(other).Swap(*this); return *this; } @@ -122,10 +122,10 @@ public: { static_assert( std::is_convertible_v<U*, T*>, - "U* must be convertible to T*"); - static_assert( - std::is_base_of_v<TRefCountedBase, T>, - "Cast allowed only for types derived from TRefCountedBase"); + "U* must be convertible to T*"); + static_assert( + std::is_base_of_v<TRefCountedBase, T>, + "Cast allowed only for types derived from TRefCountedBase"); TIntrusivePtr(std::move(other)).Swap(*this); return *this; } @@ -196,31 +196,31 @@ TIntrusivePtr<T> MakeStrong(T* p) return TIntrusivePtr<T>(p); } -//! Tries to obtain an intrusive pointer for an object that may had -//! already lost all of its references and, thus, is about to be deleted. -/*! - * You may call this method at any time provided that you have a valid - * raw pointer to an object. The call either returns an intrusive pointer - * for the object (thus ensuring that the object won't be destroyed until - * you're holding this pointer) or NULL indicating that the last reference - * had already been lost and the object is on its way to heavens. - * All these steps happen atomically. - * - * Under all circumstances it is caller's responsibility the make sure that - * the object is not destroyed during the call to #DangerousGetPtr. - * Typically this is achieved by keeping a (lock-protected) collection of - * raw pointers, taking a lock in object's destructor, and unregistering - * its raw pointer from the collection there. - */ - -template <class T> -Y_FORCE_INLINE TIntrusivePtr<T> DangerousGetPtr(T* object) -{ - return object->TryRef() - ? TIntrusivePtr<T>(object, false) - : TIntrusivePtr<T>(); -} - +//! Tries to obtain an intrusive pointer for an object that may had +//! already lost all of its references and, thus, is about to be deleted. +/*! + * You may call this method at any time provided that you have a valid + * raw pointer to an object. The call either returns an intrusive pointer + * for the object (thus ensuring that the object won't be destroyed until + * you're holding this pointer) or NULL indicating that the last reference + * had already been lost and the object is on its way to heavens. + * All these steps happen atomically. + * + * Under all circumstances it is caller's responsibility the make sure that + * the object is not destroyed during the call to #DangerousGetPtr. + * Typically this is achieved by keeping a (lock-protected) collection of + * raw pointers, taking a lock in object's destructor, and unregistering + * its raw pointer from the collection there. + */ + +template <class T> +Y_FORCE_INLINE TIntrusivePtr<T> DangerousGetPtr(T* object) +{ + return object->TryRef() + ? TIntrusivePtr<T>(object, false) + : TIntrusivePtr<T>(); +} + //////////////////////////////////////////////////////////////////////////////// template <class T, class U> @@ -272,7 +272,7 @@ bool operator==(const TIntrusivePtr<T>& lhs, const TIntrusivePtr<U>& rhs) { static_assert( std::is_convertible_v<U*, T*>, - "U* must be convertible to T*"); + "U* must be convertible to T*"); return lhs.Get() == rhs.Get(); } @@ -281,7 +281,7 @@ bool operator!=(const TIntrusivePtr<T>& lhs, const TIntrusivePtr<U>& rhs) { static_assert( std::is_convertible_v<U*, T*>, - "U* must be convertible to T*"); + "U* must be convertible to T*"); return lhs.Get() != rhs.Get(); } @@ -290,7 +290,7 @@ bool operator==(const TIntrusivePtr<T>& lhs, U* rhs) { static_assert( std::is_convertible_v<U*, T*>, - "U* must be convertible to T*"); + "U* must be convertible to T*"); return lhs.Get() == rhs; } @@ -299,7 +299,7 @@ bool operator!=(const TIntrusivePtr<T>& lhs, U* rhs) { static_assert( std::is_convertible_v<U*, T*>, - "U* must be convertible to T*"); + "U* must be convertible to T*"); return lhs.Get() != rhs; } @@ -308,7 +308,7 @@ bool operator==(T* lhs, const TIntrusivePtr<U>& rhs) { static_assert( std::is_convertible_v<U*, T*>, - "U* must be convertible to T*"); + "U* must be convertible to T*"); return lhs == rhs.Get(); } @@ -317,7 +317,7 @@ bool operator!=(T* lhs, const TIntrusivePtr<U>& rhs) { static_assert( std::is_convertible_v<U*, T*>, - "U* must be convertible to T*"); + "U* must be convertible to T*"); return lhs != rhs.Get(); } diff --git a/library/cpp/yt/memory/new-inl.h b/library/cpp/yt/memory/new-inl.h index 0a84818516..cf6e585da3 100644 --- a/library/cpp/yt/memory/new-inl.h +++ b/library/cpp/yt/memory/new-inl.h @@ -10,87 +10,87 @@ namespace NYT { //////////////////////////////////////////////////////////////////////////////// -struct TRefCountedCookieHolder -{ -#ifdef YT_ENABLE_REF_COUNTED_TRACKING - TRefCountedTypeCookie Cookie = NullRefCountedTypeCookie; - - void InitializeTracking(TRefCountedTypeCookie cookie) - { - YT_ASSERT(Cookie == NullRefCountedTypeCookie); - Cookie = cookie; - TRefCountedTrackerFacade::AllocateInstance(Cookie); - } - - ~TRefCountedCookieHolder() - { - if (Cookie != NullRefCountedTypeCookie) { - TRefCountedTrackerFacade::FreeInstance(Cookie); - } - } -#endif -}; - +struct TRefCountedCookieHolder +{ +#ifdef YT_ENABLE_REF_COUNTED_TRACKING + TRefCountedTypeCookie Cookie = NullRefCountedTypeCookie; + + void InitializeTracking(TRefCountedTypeCookie cookie) + { + YT_ASSERT(Cookie == NullRefCountedTypeCookie); + Cookie = cookie; + TRefCountedTrackerFacade::AllocateInstance(Cookie); + } + + ~TRefCountedCookieHolder() + { + if (Cookie != NullRefCountedTypeCookie) { + TRefCountedTrackerFacade::FreeInstance(Cookie); + } + } +#endif +}; + template <class T> -struct TRefCountedWrapper final - : public T - , public TRefTracked<T> -{ - template <class... TArgs> - explicit TRefCountedWrapper(TArgs&&... args) - : T(std::forward<TArgs>(args)...) - { } - - ~TRefCountedWrapper() = default; - +struct TRefCountedWrapper final + : public T + , public TRefTracked<T> +{ + template <class... TArgs> + explicit TRefCountedWrapper(TArgs&&... args) + : T(std::forward<TArgs>(args)...) + { } + + ~TRefCountedWrapper() = default; + void DestroyRefCounted() override - { - T::DestroyRefCountedImpl(this); - } -}; - -template <class T, class TDeleter> -class TRefCountedWrapperWithDeleter final - : public T - , public TRefTracked<T> -{ -public: - template <class... TArgs> - explicit TRefCountedWrapperWithDeleter(const TDeleter& deleter, TArgs&&... args) - : T(std::forward<TArgs>(args)...) - , Deleter_(deleter) - { } - - ~TRefCountedWrapperWithDeleter() = default; - + { + T::DestroyRefCountedImpl(this); + } +}; + +template <class T, class TDeleter> +class TRefCountedWrapperWithDeleter final + : public T + , public TRefTracked<T> +{ +public: + template <class... TArgs> + explicit TRefCountedWrapperWithDeleter(const TDeleter& deleter, TArgs&&... args) + : T(std::forward<TArgs>(args)...) + , Deleter_(deleter) + { } + + ~TRefCountedWrapperWithDeleter() = default; + void DestroyRefCounted() override - { - Deleter_(this); - } - -private: - TDeleter Deleter_; -}; - -template <class T> -struct TRefCountedWrapperWithCookie final - : public T - , public TRefCountedCookieHolder -{ - template <class... TArgs> - explicit TRefCountedWrapperWithCookie(TArgs&&... args) - : T(std::forward<TArgs>(args)...) - { } - - ~TRefCountedWrapperWithCookie() = default; - + { + Deleter_(this); + } + +private: + TDeleter Deleter_; +}; + +template <class T> +struct TRefCountedWrapperWithCookie final + : public T + , public TRefCountedCookieHolder +{ + template <class... TArgs> + explicit TRefCountedWrapperWithCookie(TArgs&&... args) + : T(std::forward<TArgs>(args)...) + { } + + ~TRefCountedWrapperWithCookie() = default; + void DestroyRefCounted() override - { - T::DestroyRefCountedImpl(this); - } -}; + { + T::DestroyRefCountedImpl(this); + } +}; -namespace NDetail { +namespace NDetail { Y_FORCE_INLINE void* AllignedMalloc(size_t size, size_t allignment) { @@ -115,180 +115,180 @@ Y_FORCE_INLINE auto CustomInitialize(T* ptr) -> decltype(&T::InitializeRefCounte ptr->InitializeRefCounted(); } -template <class T, class... As> -Y_FORCE_INLINE T* NewEpilogue(void* ptr, As&& ... args) -{ - try { - auto* instance = static_cast<T*>(ptr); - new (instance) T(std::forward<As>(args)...); +template <class T, class... As> +Y_FORCE_INLINE T* NewEpilogue(void* ptr, As&& ... args) +{ + try { + auto* instance = static_cast<T*>(ptr); + new (instance) T(std::forward<As>(args)...); CustomInitialize(instance); - return instance; - } catch (const std::exception& ex) { - // Do not forget to free the memory. - TFreeMemory<T>::Do(ptr); - throw; - } -} - -template <class T, bool = std::is_base_of_v<TRefCountedBase, T>> -struct TConstructHelper -{ - static constexpr size_t RefCounterSpace = (sizeof(TRefCounter) + alignof(T) - 1) & ~(alignof(T) - 1); - static constexpr size_t RefCounterOffset = RefCounterSpace - sizeof(TRefCounter); - static constexpr size_t Size = RefCounterSpace + sizeof(T); - static constexpr size_t Alignment = alignof(T); - - template <class... As> - Y_FORCE_INLINE static T* Construct(void* ptr, As&&... args) - { - auto* refCounter = reinterpret_cast<TRefCounter*>(static_cast<char*>(ptr) + RefCounterOffset); - new (refCounter) TRefCounter(); - auto* object = reinterpret_cast<T*>(refCounter + 1); - if constexpr (std::is_constructible_v<T, As...>) { - new(object) T(std::forward<As>(args)...); - } else { - new(object) T{std::forward<As>(args)...}; - } - CustomInitialize(object); - return object; - } -}; - -template <class T> -struct TConstructHelper<T, true> -{ - static constexpr size_t Size = sizeof(TRefCountedWrapper<T>); - static constexpr size_t Alignment = alignof(TRefCountedWrapper<T>); - - template <class... As> - Y_FORCE_INLINE static TRefCountedWrapper<T>* Construct(void* ptr, As&&... args) - { - using TDerived = TRefCountedWrapper<T>; - auto* object = new(static_cast<TDerived*>(ptr)) TDerived(std::forward<As>(args)...); - CustomInitialize(object); - return object; - } -}; - + return instance; + } catch (const std::exception& ex) { + // Do not forget to free the memory. + TFreeMemory<T>::Do(ptr); + throw; + } +} + +template <class T, bool = std::is_base_of_v<TRefCountedBase, T>> +struct TConstructHelper +{ + static constexpr size_t RefCounterSpace = (sizeof(TRefCounter) + alignof(T) - 1) & ~(alignof(T) - 1); + static constexpr size_t RefCounterOffset = RefCounterSpace - sizeof(TRefCounter); + static constexpr size_t Size = RefCounterSpace + sizeof(T); + static constexpr size_t Alignment = alignof(T); + + template <class... As> + Y_FORCE_INLINE static T* Construct(void* ptr, As&&... args) + { + auto* refCounter = reinterpret_cast<TRefCounter*>(static_cast<char*>(ptr) + RefCounterOffset); + new (refCounter) TRefCounter(); + auto* object = reinterpret_cast<T*>(refCounter + 1); + if constexpr (std::is_constructible_v<T, As...>) { + new(object) T(std::forward<As>(args)...); + } else { + new(object) T{std::forward<As>(args)...}; + } + CustomInitialize(object); + return object; + } +}; + +template <class T> +struct TConstructHelper<T, true> +{ + static constexpr size_t Size = sizeof(TRefCountedWrapper<T>); + static constexpr size_t Alignment = alignof(TRefCountedWrapper<T>); + + template <class... As> + Y_FORCE_INLINE static TRefCountedWrapper<T>* Construct(void* ptr, As&&... args) + { + using TDerived = TRefCountedWrapper<T>; + auto* object = new(static_cast<TDerived*>(ptr)) TDerived(std::forward<As>(args)...); + CustomInitialize(object); + return object; + } +}; + template <class T, class... As> -Y_FORCE_INLINE TIntrusivePtr<T> SafeConstruct(void* ptr, As&&... args) +Y_FORCE_INLINE TIntrusivePtr<T> SafeConstruct(void* ptr, As&&... args) { try { - auto* instance = TConstructHelper<T>::Construct(ptr, std::forward<As>(args)...); - return TIntrusivePtr<T>(instance, false); + auto* instance = TConstructHelper<T>::Construct(ptr, std::forward<As>(args)...); + return TIntrusivePtr<T>(instance, false); } catch (const std::exception& ex) { // Do not forget to free the memory. - TFreeMemory<T>::Do(ptr); + TFreeMemory<T>::Do(ptr); throw; } } -template <size_t Size, size_t Alignment> -void* AllocateConstSizeAligned() -{ - if (Alignment <= 16) { - return NYTAlloc::AllocateConstSize<Size>(); - } else { +template <size_t Size, size_t Alignment> +void* AllocateConstSizeAligned() +{ + if (Alignment <= 16) { + return NYTAlloc::AllocateConstSize<Size>(); + } else { return AllignedMalloc(Size, Alignment); - } -} - + } +} + } // namespace NDetail -//////////////////////////////////////////////////////////////////////////////// - -template <class T, class... As, class> -Y_FORCE_INLINE TIntrusivePtr<T> New( - As&&... args) -{ - void* ptr = NDetail::AllocateConstSizeAligned< - NDetail::TConstructHelper<T>::Size, - NDetail::TConstructHelper<T>::Alignment>(); - - return NDetail::SafeConstruct<T>(ptr, std::forward<As>(args)...); -} - -template <class T, class... As, class> -Y_FORCE_INLINE TIntrusivePtr<T> New( - typename T::TAllocator* allocator, - As&&... args) +//////////////////////////////////////////////////////////////////////////////// + +template <class T, class... As, class> +Y_FORCE_INLINE TIntrusivePtr<T> New( + As&&... args) { - auto* ptr = allocator->Allocate(NDetail::TConstructHelper<T>::Size); - if (!ptr) { - return nullptr; - } - return NDetail::SafeConstruct<T>(ptr, std::forward<As>(args)...); + void* ptr = NDetail::AllocateConstSizeAligned< + NDetail::TConstructHelper<T>::Size, + NDetail::TConstructHelper<T>::Alignment>(); + + return NDetail::SafeConstruct<T>(ptr, std::forward<As>(args)...); +} + +template <class T, class... As, class> +Y_FORCE_INLINE TIntrusivePtr<T> New( + typename T::TAllocator* allocator, + As&&... args) +{ + auto* ptr = allocator->Allocate(NDetail::TConstructHelper<T>::Size); + if (!ptr) { + return nullptr; + } + return NDetail::SafeConstruct<T>(ptr, std::forward<As>(args)...); } -//////////////////////////////////////////////////////////////////////////////// - -template <class T, class... As, class> -Y_FORCE_INLINE TIntrusivePtr<T> NewWithExtraSpace( +//////////////////////////////////////////////////////////////////////////////// + +template <class T, class... As, class> +Y_FORCE_INLINE TIntrusivePtr<T> NewWithExtraSpace( size_t extraSpaceSize, As&&... args) { auto totalSize = NYT::NDetail::TConstructHelper<T>::Size + extraSpaceSize; - void* ptr = nullptr; - + void* ptr = nullptr; + if (NYT::NDetail::TConstructHelper<T>::Alignment <= 16) { - ptr = NYTAlloc::Allocate(totalSize); - } else { + ptr = NYTAlloc::Allocate(totalSize); + } else { ptr = NYT::NDetail::AllignedMalloc(totalSize, NYT::NDetail::TConstructHelper<T>::Alignment); - } - + } + return NYT::NDetail::SafeConstruct<T>(ptr, std::forward<As>(args)...); -} - -template <class T, class... As, class> -Y_FORCE_INLINE TIntrusivePtr<T> NewWithExtraSpace( - typename T::TAllocator* allocator, - size_t extraSpaceSize, - As&&... args) -{ +} + +template <class T, class... As, class> +Y_FORCE_INLINE TIntrusivePtr<T> NewWithExtraSpace( + typename T::TAllocator* allocator, + size_t extraSpaceSize, + As&&... args) +{ auto totalSize = NYT::NDetail::TConstructHelper<T>::Size + extraSpaceSize; - auto* ptr = allocator->Allocate(totalSize); - if (!ptr) { - return nullptr; - } + auto* ptr = allocator->Allocate(totalSize); + if (!ptr) { + return nullptr; + } return NYT::NDetail::SafeConstruct<T>(ptr, std::forward<As>(args)...); +} + +//////////////////////////////////////////////////////////////////////////////// + +// Support for polymorphic only +template <class T, class TDeleter, class... As> +Y_FORCE_INLINE TIntrusivePtr<T> NewWithDelete(const TDeleter& deleter, As&&... args) +{ + using TWrapper = TRefCountedWrapperWithDeleter<T, TDeleter>; + void* ptr = NDetail::AllocateConstSizeAligned<sizeof(TWrapper), alignof(TWrapper)>(); + + auto* instance = NDetail::NewEpilogue<TWrapper>( + ptr, + deleter, + std::forward<As>(args)...); + + return TIntrusivePtr<T>(instance, false); } -//////////////////////////////////////////////////////////////////////////////// - -// Support for polymorphic only -template <class T, class TDeleter, class... As> -Y_FORCE_INLINE TIntrusivePtr<T> NewWithDelete(const TDeleter& deleter, As&&... args) -{ - using TWrapper = TRefCountedWrapperWithDeleter<T, TDeleter>; - void* ptr = NDetail::AllocateConstSizeAligned<sizeof(TWrapper), alignof(TWrapper)>(); - - auto* instance = NDetail::NewEpilogue<TWrapper>( - ptr, - deleter, - std::forward<As>(args)...); - - return TIntrusivePtr<T>(instance, false); -} - -//////////////////////////////////////////////////////////////////////////////// - +//////////////////////////////////////////////////////////////////////////////// + template <class T, class TTag, int Counter, class... As> Y_FORCE_INLINE TIntrusivePtr<T> NewWithLocation( const TSourceLocation& location, As&&... args) { - using TWrapper = TRefCountedWrapperWithCookie<T>; - void* ptr = NDetail::AllocateConstSizeAligned<sizeof(TWrapper), alignof(TWrapper)>(); - - auto* instance = NDetail::NewEpilogue<TWrapper>(ptr, std::forward<As>(args)...); - + using TWrapper = TRefCountedWrapperWithCookie<T>; + void* ptr = NDetail::AllocateConstSizeAligned<sizeof(TWrapper), alignof(TWrapper)>(); + + auto* instance = NDetail::NewEpilogue<TWrapper>(ptr, std::forward<As>(args)...); + #ifdef YT_ENABLE_REF_COUNTED_TRACKING - instance->InitializeTracking(GetRefCountedTypeCookieWithLocation<T, TTag, Counter>(location)); + instance->InitializeTracking(GetRefCountedTypeCookieWithLocation<T, TTag, Counter>(location)); #else - Y_UNUSED(location); + Y_UNUSED(location); #endif - - return TIntrusivePtr<T>(instance, false); + + return TIntrusivePtr<T>(instance, false); } //////////////////////////////////////////////////////////////////////////////// diff --git a/library/cpp/yt/memory/new.h b/library/cpp/yt/memory/new.h index 2db45e0465..1b448df993 100644 --- a/library/cpp/yt/memory/new.h +++ b/library/cpp/yt/memory/new.h @@ -1,10 +1,10 @@ #pragma once -#include "intrusive_ptr.h" -#include "ref_tracked.h" +#include "intrusive_ptr.h" +#include "ref_tracked.h" #include <library/cpp/yt/misc/source_location.h> - + #include <util/system/defaults.h> namespace NYT { @@ -65,47 +65,47 @@ namespace NYT { //////////////////////////////////////////////////////////////////////////////// -template <class T, class = void> -struct THasAllocator -{ - using TFalse = void; -}; - -template <class T> +template <class T, class = void> +struct THasAllocator +{ + using TFalse = void; +}; + +template <class T> struct THasAllocator<T, std::void_t<typename T::TAllocator>> -{ - using TTrue = void; -}; - -//////////////////////////////////////////////////////////////////////////////// - +{ + using TTrue = void; +}; + +//////////////////////////////////////////////////////////////////////////////// + //! Allocates a new instance of |T|. -template <class T, class... As, class = typename THasAllocator<T>::TFalse> +template <class T, class... As, class = typename THasAllocator<T>::TFalse> TIntrusivePtr<T> New(As&&... args); -template <class T, class... As, class = typename THasAllocator<T>::TTrue> -TIntrusivePtr<T> New(typename T::TAllocator* allocator, As&&... args); - -//! Allocates an instance of |T| with additional storage of #extraSpaceSize bytes. -template <class T, class... As, class = typename THasAllocator<T>::TFalse> -TIntrusivePtr<T> NewWithExtraSpace(size_t extraSpaceSize, As&&... args); - -template <class T, class... As, class = typename THasAllocator<T>::TTrue> -TIntrusivePtr<T> NewWithExtraSpace(typename T::TAllocator* allocator, size_t extraSpaceSize, As&&... args); - -//! Allocates a new instance of |T| with user deleter. -template <class T, class TDeleter, class... As> -TIntrusivePtr<T> NewWithDelete(const TDeleter& deleter, As&&... args); - +template <class T, class... As, class = typename THasAllocator<T>::TTrue> +TIntrusivePtr<T> New(typename T::TAllocator* allocator, As&&... args); + +//! Allocates an instance of |T| with additional storage of #extraSpaceSize bytes. +template <class T, class... As, class = typename THasAllocator<T>::TFalse> +TIntrusivePtr<T> NewWithExtraSpace(size_t extraSpaceSize, As&&... args); + +template <class T, class... As, class = typename THasAllocator<T>::TTrue> +TIntrusivePtr<T> NewWithExtraSpace(typename T::TAllocator* allocator, size_t extraSpaceSize, As&&... args); + +//! Allocates a new instance of |T| with user deleter. +template <class T, class TDeleter, class... As> +TIntrusivePtr<T> NewWithDelete(const TDeleter& deleter, As&&... args); + //! Allocates a new instance of |T|. //! The allocation is additionally marked with #location. template <class T, class TTag, int Counter, class... As> -TIntrusivePtr<T> NewWithLocation(const TSourceLocation& location, As&&... args); +TIntrusivePtr<T> NewWithLocation(const TSourceLocation& location, As&&... args); //! Enables calling #New and co for types with private ctors. #define DECLARE_NEW_FRIEND() \ - template <class DECLARE_NEW_FRIEND_T> \ - friend struct NYT::TRefCountedWrapper; + template <class DECLARE_NEW_FRIEND_T> \ + friend struct NYT::TRefCountedWrapper; //////////////////////////////////////////////////////////////////////////////// diff --git a/library/cpp/yt/memory/range.h b/library/cpp/yt/memory/range.h index 6c71aa9496..49ca05a61e 100644 --- a/library/cpp/yt/memory/range.h +++ b/library/cpp/yt/memory/range.h @@ -187,7 +187,7 @@ public: TRange<T> Slice(size_t startOffset, size_t endOffset) const { - YT_ASSERT(startOffset <= endOffset && endOffset <= Size()); + YT_ASSERT(startOffset <= endOffset && endOffset <= Size()); return TRange<T>(Begin() + startOffset, endOffset - startOffset); } @@ -361,7 +361,7 @@ public: using TRange<T>::Front; using TRange<T>::Back; using TRange<T>::operator[]; - + iterator Begin() const { return const_cast<T*>(this->Data_); @@ -402,12 +402,12 @@ public: return Begin()[this->Length_ - 1]; } - TMutableRange<T> Slice(size_t startOffset, size_t endOffset) const - { - YT_ASSERT(startOffset <= endOffset && endOffset <= this->Size()); - return TMutableRange<T>(Begin() + startOffset, endOffset - startOffset); - } - + TMutableRange<T> Slice(size_t startOffset, size_t endOffset) const + { + YT_ASSERT(startOffset <= endOffset && endOffset <= this->Size()); + return TMutableRange<T>(Begin() + startOffset, endOffset - startOffset); + } + TMutableRange<T> Slice(T* begin, T* end) const { YT_ASSERT(begin >= Begin()); diff --git a/library/cpp/yt/memory/ref.cpp b/library/cpp/yt/memory/ref.cpp index e8ff42e976..10d253f16a 100644 --- a/library/cpp/yt/memory/ref.cpp +++ b/library/cpp/yt/memory/ref.cpp @@ -20,87 +20,87 @@ char MutableEmptyRefData[1] = {0}; //////////////////////////////////////////////////////////////////////////////// -class TBlobHolder - : public TRefCounted -{ -public: - explicit TBlobHolder(TBlob&& blob) - : Blob_(std::move(blob)) - { } - -private: - const TBlob Blob_; -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TStringHolder - : public TRefCounted -{ -public: - TStringHolder(TString&& string, TRefCountedTypeCookie cookie) - : String_(std::move(string)) -#ifdef YT_ENABLE_REF_COUNTED_TRACKING - , Cookie_(cookie) -#endif - { -#ifdef YT_ENABLE_REF_COUNTED_TRACKING - TRefCountedTrackerFacade::AllocateTagInstance(Cookie_); - TRefCountedTrackerFacade::AllocateSpace(Cookie_, String_.length()); -#endif - } - ~TStringHolder() - { -#ifdef YT_ENABLE_REF_COUNTED_TRACKING - TRefCountedTrackerFacade::FreeTagInstance(Cookie_); - TRefCountedTrackerFacade::FreeSpace(Cookie_, String_.length()); -#endif - } - +class TBlobHolder + : public TRefCounted +{ +public: + explicit TBlobHolder(TBlob&& blob) + : Blob_(std::move(blob)) + { } + +private: + const TBlob Blob_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TStringHolder + : public TRefCounted +{ +public: + TStringHolder(TString&& string, TRefCountedTypeCookie cookie) + : String_(std::move(string)) +#ifdef YT_ENABLE_REF_COUNTED_TRACKING + , Cookie_(cookie) +#endif + { +#ifdef YT_ENABLE_REF_COUNTED_TRACKING + TRefCountedTrackerFacade::AllocateTagInstance(Cookie_); + TRefCountedTrackerFacade::AllocateSpace(Cookie_, String_.length()); +#endif + } + ~TStringHolder() + { +#ifdef YT_ENABLE_REF_COUNTED_TRACKING + TRefCountedTrackerFacade::FreeTagInstance(Cookie_); + TRefCountedTrackerFacade::FreeSpace(Cookie_, String_.length()); +#endif + } + const TString& String() const { return String_; } -private: - const TString String_; -#ifdef YT_ENABLE_REF_COUNTED_TRACKING - const TRefCountedTypeCookie Cookie_; -#endif -}; - -//////////////////////////////////////////////////////////////////////////////// - +private: + const TString String_; +#ifdef YT_ENABLE_REF_COUNTED_TRACKING + const TRefCountedTypeCookie Cookie_; +#endif +}; + +//////////////////////////////////////////////////////////////////////////////// + template <class TDerived> class TAllocationHolderBase - : public TRefCounted -{ -public: + : public TRefCounted +{ +public: TAllocationHolderBase(size_t size, TRefCountedTypeCookie cookie) - : Size_(size) -#ifdef YT_ENABLE_REF_COUNTED_TRACKING - , Cookie_(cookie) -#endif + : Size_(size) +#ifdef YT_ENABLE_REF_COUNTED_TRACKING + , Cookie_(cookie) +#endif { } ~TAllocationHolderBase() - { -#ifdef YT_ENABLE_REF_COUNTED_TRACKING - TRefCountedTrackerFacade::FreeTagInstance(Cookie_); - TRefCountedTrackerFacade::FreeSpace(Cookie_, Size_); -#endif - } - - TMutableRef GetRef() - { + { +#ifdef YT_ENABLE_REF_COUNTED_TRACKING + TRefCountedTrackerFacade::FreeTagInstance(Cookie_); + TRefCountedTrackerFacade::FreeSpace(Cookie_, Size_); +#endif + } + + TMutableRef GetRef() + { return TMutableRef(static_cast<TDerived*>(this)->GetBegin(), Size_); - } - + } + protected: - const size_t Size_; -#ifdef YT_ENABLE_REF_COUNTED_TRACKING - const TRefCountedTypeCookie Cookie_; -#endif + const size_t Size_; +#ifdef YT_ENABLE_REF_COUNTED_TRACKING + const TRefCountedTypeCookie Cookie_; +#endif void Initialize(bool initializeStorage) { @@ -112,16 +112,16 @@ protected: TRefCountedTrackerFacade::AllocateSpace(Cookie_, Size_); #endif } -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TDefaultAllocationHolder - : public TAllocationHolderBase<TDefaultAllocationHolder> - , public TWithExtraSpace<TDefaultAllocationHolder> +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TDefaultAllocationHolder + : public TAllocationHolderBase<TDefaultAllocationHolder> + , public TWithExtraSpace<TDefaultAllocationHolder> { public: - TDefaultAllocationHolder(size_t size, bool initializeStorage, TRefCountedTypeCookie cookie) + TDefaultAllocationHolder(size_t size, bool initializeStorage, TRefCountedTypeCookie cookie) : TAllocationHolderBase(size, cookie) { Initialize(initializeStorage); @@ -206,12 +206,12 @@ TSharedRef TSharedRef::MakeCopy(TRef ref, TRefCountedTypeCookie tagCookie) if (!ref) { return {}; } - if (ref.Empty()) { + if (ref.Empty()) { return TSharedRef::MakeEmpty(); - } - auto result = TSharedMutableRef::Allocate(ref.Size(), false, tagCookie); - ::memcpy(result.Begin(), ref.Begin(), ref.Size()); - return result; + } + auto result = TSharedMutableRef::Allocate(ref.Size(), false, tagCookie); + ::memcpy(result.Begin(), ref.Begin(), ref.Size()); + return result; } std::vector<TSharedRef> TSharedRef::Split(size_t partSize) const @@ -235,7 +235,7 @@ std::vector<TSharedRef> TSharedRef::Split(size_t partSize) const TSharedMutableRef TSharedMutableRef::Allocate(size_t size, bool initializeStorage, TRefCountedTypeCookie tagCookie) { - auto holder = NewWithExtraSpace<TDefaultAllocationHolder>(size, size, initializeStorage, tagCookie); + auto holder = NewWithExtraSpace<TDefaultAllocationHolder>(size, size, initializeStorage, tagCookie); auto ref = holder->GetRef(); return TSharedMutableRef(ref, std::move(holder)); } @@ -259,12 +259,12 @@ TSharedMutableRef TSharedMutableRef::MakeCopy(TRef ref, TRefCountedTypeCookie ta if (!ref) { return {}; } - if (ref.Empty()) { + if (ref.Empty()) { return TSharedMutableRef::MakeEmpty(); } - auto result = Allocate(ref.Size(), false, tagCookie); - ::memcpy(result.Begin(), ref.Begin(), ref.Size()); - return result; + auto result = Allocate(ref.Size(), false, tagCookie); + ::memcpy(result.Begin(), ref.Begin(), ref.Size()); + return result; } //////////////////////////////////////////////////////////////////////////////// @@ -295,13 +295,13 @@ size_t GetPageSize() return PageSize; } -size_t RoundUpToPage(size_t bytes) -{ - static const size_t PageSize = NSystemInfo::GetPageSize(); +size_t RoundUpToPage(size_t bytes) +{ + static const size_t PageSize = NSystemInfo::GetPageSize(); YT_ASSERT((PageSize & (PageSize - 1)) == 0); - return (bytes + PageSize - 1) & (~(PageSize - 1)); -} - + return (bytes + PageSize - 1) & (~(PageSize - 1)); +} + size_t GetByteSize(const TSharedRefArray& array) { size_t size = 0; @@ -362,7 +362,7 @@ TMutableRef TSharedRefArrayBuilder::AllocateAndAdd(size_t size) YT_ASSERT(CurrentAllocationPtr_ + size <= Impl_->GetBeginAllocationPtr() + AllocationCapacity_); TMutableRef ref(CurrentAllocationPtr_, size); CurrentAllocationPtr_ += size; - TRefCountedPtr holder(Impl_.Get(), false); + TRefCountedPtr holder(Impl_.Get(), false); TSharedRef sharedRef(ref, std::move(holder)); Add(std::move(sharedRef)); return ref; diff --git a/library/cpp/yt/memory/ref.h b/library/cpp/yt/memory/ref.h index 73d19d9013..3fde728e0f 100644 --- a/library/cpp/yt/memory/ref.h +++ b/library/cpp/yt/memory/ref.h @@ -366,8 +366,8 @@ TString ToString(const TSharedRef& ref); TString ToString(const TSharedMutableRef& ref); size_t GetPageSize(); -size_t RoundUpToPage(size_t bytes); - +size_t RoundUpToPage(size_t bytes); + size_t GetByteSize(TRef ref); size_t GetByteSize(const TSharedRefArray& array); template <class T> @@ -375,8 +375,8 @@ size_t GetByteSize(TRange<T> parts); template <class T> size_t GetByteSize(const std::vector<T>& parts); -//////////////////////////////////////////////////////////////////////////////// - +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT #define REF_INL_H_ diff --git a/library/cpp/yt/memory/ref_counted-inl.h b/library/cpp/yt/memory/ref_counted-inl.h index e6d64fec18..6b9bfeca8c 100644 --- a/library/cpp/yt/memory/ref_counted-inl.h +++ b/library/cpp/yt/memory/ref_counted-inl.h @@ -8,269 +8,269 @@ namespace NYT { //////////////////////////////////////////////////////////////////////////////// -constexpr uint16_t PtrBits = 48; -constexpr uintptr_t PtrMask = (1ULL << PtrBits) - 1; - -template <class T> -Y_FORCE_INLINE char* PackPointer(T* ptr, uint16_t data) +constexpr uint16_t PtrBits = 48; +constexpr uintptr_t PtrMask = (1ULL << PtrBits) - 1; + +template <class T> +Y_FORCE_INLINE char* PackPointer(T* ptr, uint16_t data) { - return reinterpret_cast<char*>((static_cast<uintptr_t>(data) << PtrBits) | reinterpret_cast<uintptr_t>(ptr)); + return reinterpret_cast<char*>((static_cast<uintptr_t>(data) << PtrBits) | reinterpret_cast<uintptr_t>(ptr)); } -template <class T> -struct TPackedPointer -{ - uint16_t Data; - T* Ptr; -}; - -template <class T> -Y_FORCE_INLINE TPackedPointer<T> UnpackPointer(void* packedPtr) -{ - auto castedPtr = reinterpret_cast<uintptr_t>(packedPtr); - return {static_cast<uint16_t>(castedPtr >> PtrBits), reinterpret_cast<T*>(castedPtr & PtrMask)}; -} - -//////////////////////////////////////////////////////////////////////////////// - -template <class T, class = void> -struct TMemoryReleaser +template <class T> +struct TPackedPointer +{ + uint16_t Data; + T* Ptr; +}; + +template <class T> +Y_FORCE_INLINE TPackedPointer<T> UnpackPointer(void* packedPtr) +{ + auto castedPtr = reinterpret_cast<uintptr_t>(packedPtr); + return {static_cast<uint16_t>(castedPtr >> PtrBits), reinterpret_cast<T*>(castedPtr & PtrMask)}; +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class T, class = void> +struct TMemoryReleaser { - static void Do(void* ptr, uint16_t /*offset*/) - { - TFreeMemory<T>::Do(ptr); - } -}; - -using TDeleter = void (*)(void*); - -void ScheduleObjectDeletion(void* ptr, TDeleter deleter); - -template <class T> + static void Do(void* ptr, uint16_t /*offset*/) + { + TFreeMemory<T>::Do(ptr); + } +}; + +using TDeleter = void (*)(void*); + +void ScheduleObjectDeletion(void* ptr, TDeleter deleter); + +template <class T> struct TMemoryReleaser<T, std::enable_if_t<T::EnableHazard>> -{ - static void Do(void* ptr, uint16_t offset) - { - // Base pointer is used in HazardPtr as the identity of object. - auto* basePtr = PackPointer(static_cast<char*>(ptr) + offset, offset); - - ScheduleObjectDeletion(basePtr, [] (void* ptr) { - // Base ptr and the beginning of allocated memory region may differ. - auto [offset, basePtr] = UnpackPointer<char>(ptr); - TFreeMemory<T>::Do(basePtr - offset); - }); - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -Y_FORCE_INLINE int TRefCounter::GetRefCount() const noexcept -{ - return StrongCount_.load(std::memory_order_relaxed); -} - -Y_FORCE_INLINE void TRefCounter::Ref() const noexcept -{ +{ + static void Do(void* ptr, uint16_t offset) + { + // Base pointer is used in HazardPtr as the identity of object. + auto* basePtr = PackPointer(static_cast<char*>(ptr) + offset, offset); + + ScheduleObjectDeletion(basePtr, [] (void* ptr) { + // Base ptr and the beginning of allocated memory region may differ. + auto [offset, basePtr] = UnpackPointer<char>(ptr); + TFreeMemory<T>::Do(basePtr - offset); + }); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +Y_FORCE_INLINE int TRefCounter::GetRefCount() const noexcept +{ + return StrongCount_.load(std::memory_order_relaxed); +} + +Y_FORCE_INLINE void TRefCounter::Ref() const noexcept +{ // It is safe to use relaxed here, since new reference is always created from another live reference. - StrongCount_.fetch_add(1, std::memory_order_relaxed); - - YT_ASSERT(WeakCount_.load(std::memory_order_relaxed) > 0); + StrongCount_.fetch_add(1, std::memory_order_relaxed); + + YT_ASSERT(WeakCount_.load(std::memory_order_relaxed) > 0); } -Y_FORCE_INLINE bool TRefCounter::TryRef() const noexcept +Y_FORCE_INLINE bool TRefCounter::TryRef() const noexcept { - auto value = StrongCount_.load(std::memory_order_relaxed); - YT_ASSERT(WeakCount_.load(std::memory_order_relaxed) > 0); - - while (value != 0 && !StrongCount_.compare_exchange_weak(value, value + 1)); - return value != 0; -} - -Y_FORCE_INLINE bool TRefCounter::Unref() const -{ - // We must properly synchronize last access to object with it destruction. - // Otherwise compiler might reorder access to object past this decrement. - // - // See http://www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html#boost_atomic.usage_examples.example_reference_counters - // + auto value = StrongCount_.load(std::memory_order_relaxed); + YT_ASSERT(WeakCount_.load(std::memory_order_relaxed) > 0); + + while (value != 0 && !StrongCount_.compare_exchange_weak(value, value + 1)); + return value != 0; +} + +Y_FORCE_INLINE bool TRefCounter::Unref() const +{ + // We must properly synchronize last access to object with it destruction. + // Otherwise compiler might reorder access to object past this decrement. + // + // See http://www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html#boost_atomic.usage_examples.example_reference_counters + // auto oldStrongCount = StrongCount_.fetch_sub(1, std::memory_order_release); YT_ASSERT(oldStrongCount > 0); - if (oldStrongCount == 1) { - StrongCount_.load(std::memory_order_acquire); - return true; - } else { - return false; - } + if (oldStrongCount == 1) { + StrongCount_.load(std::memory_order_acquire); + return true; + } else { + return false; + } +} + +Y_FORCE_INLINE int TRefCounter::GetWeakRefCount() const noexcept +{ + return WeakCount_.load(std::memory_order_acquire); +} + +Y_FORCE_INLINE void TRefCounter::WeakRef() const noexcept +{ + auto oldWeakCount = WeakCount_.fetch_add(1, std::memory_order_relaxed); + YT_ASSERT(oldWeakCount > 0); } -Y_FORCE_INLINE int TRefCounter::GetWeakRefCount() const noexcept +Y_FORCE_INLINE bool TRefCounter::WeakUnref() const +{ + auto oldWeakCount = WeakCount_.fetch_sub(1, std::memory_order_release); + YT_ASSERT(oldWeakCount > 0); + if (oldWeakCount == 1) { + WeakCount_.load(std::memory_order_acquire); + return true; + } else { + return false; + } +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class T, bool = std::is_base_of_v<TRefCountedBase, T>> +struct TRefCountedHelper { - return WeakCount_.load(std::memory_order_acquire); -} - -Y_FORCE_INLINE void TRefCounter::WeakRef() const noexcept -{ - auto oldWeakCount = WeakCount_.fetch_add(1, std::memory_order_relaxed); - YT_ASSERT(oldWeakCount > 0); -} - -Y_FORCE_INLINE bool TRefCounter::WeakUnref() const -{ - auto oldWeakCount = WeakCount_.fetch_sub(1, std::memory_order_release); - YT_ASSERT(oldWeakCount > 0); - if (oldWeakCount == 1) { - WeakCount_.load(std::memory_order_acquire); - return true; - } else { - return false; - } -} - -//////////////////////////////////////////////////////////////////////////////// - -template <class T, bool = std::is_base_of_v<TRefCountedBase, T>> -struct TRefCountedHelper -{ - static_assert( - std::is_final_v<T>, - "Ref-counted objects must be derived from TRefCountedBase or to be final"); - - static constexpr size_t RefCounterSpace = (sizeof(TRefCounter) + alignof(T) - 1) & ~(alignof(T) - 1); - static constexpr size_t RefCounterOffset = RefCounterSpace - sizeof(TRefCounter); - - Y_FORCE_INLINE static const TRefCounter* GetRefCounter(const T* obj) - { - return reinterpret_cast<const TRefCounter*>(obj) - 1; - } + static_assert( + std::is_final_v<T>, + "Ref-counted objects must be derived from TRefCountedBase or to be final"); + + static constexpr size_t RefCounterSpace = (sizeof(TRefCounter) + alignof(T) - 1) & ~(alignof(T) - 1); + static constexpr size_t RefCounterOffset = RefCounterSpace - sizeof(TRefCounter); + + Y_FORCE_INLINE static const TRefCounter* GetRefCounter(const T* obj) + { + return reinterpret_cast<const TRefCounter*>(obj) - 1; + } Y_FORCE_INLINE static void Destroy(const T* obj) - { - auto* refCounter = GetRefCounter(obj); - - // No virtual call when T is final. - obj->~T(); - - char* ptr = reinterpret_cast<char*>(const_cast<TRefCounter*>(refCounter)); - - // Fast path. Weak refs cannot appear if there are neither strong nor weak refs. - if (refCounter->GetWeakRefCount() == 1) { - TMemoryReleaser<T>::Do(ptr - RefCounterOffset, RefCounterSpace); - return; - } - - if (refCounter->WeakUnref()) { - TMemoryReleaser<T>::Do(ptr - RefCounterOffset, RefCounterSpace); - } - } - + { + auto* refCounter = GetRefCounter(obj); + + // No virtual call when T is final. + obj->~T(); + + char* ptr = reinterpret_cast<char*>(const_cast<TRefCounter*>(refCounter)); + + // Fast path. Weak refs cannot appear if there are neither strong nor weak refs. + if (refCounter->GetWeakRefCount() == 1) { + TMemoryReleaser<T>::Do(ptr - RefCounterOffset, RefCounterSpace); + return; + } + + if (refCounter->WeakUnref()) { + TMemoryReleaser<T>::Do(ptr - RefCounterOffset, RefCounterSpace); + } + } + Y_FORCE_INLINE static void Deallocate(const T* obj) - { - char* ptr = reinterpret_cast<char*>(const_cast<TRefCounter*>(GetRefCounter(obj))); - TMemoryReleaser<T>::Do(ptr - RefCounterOffset, RefCounterSpace); - } -}; - -template <class T> -struct TRefCountedHelper<T, true> -{ - Y_FORCE_INLINE static const TRefCounter* GetRefCounter(const T* obj) - { - return obj; - } - - Y_FORCE_INLINE static void Destroy(const TRefCountedBase* obj) - { - const_cast<TRefCountedBase*>(obj)->DestroyRefCounted(); - } - - Y_FORCE_INLINE static void Deallocate(const TRefCountedBase* obj) - { - auto* ptr = reinterpret_cast<void**>(const_cast<TRefCountedBase*>(obj)); - auto [offset, ptrToDeleter] = UnpackPointer<void(void*, uint16_t)>(*ptr); - - // The most derived type is erased here. So we cannot call TMemoryReleaser with derived type. - ptrToDeleter(reinterpret_cast<char*>(ptr) - offset, offset); - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -template <class T> + { + char* ptr = reinterpret_cast<char*>(const_cast<TRefCounter*>(GetRefCounter(obj))); + TMemoryReleaser<T>::Do(ptr - RefCounterOffset, RefCounterSpace); + } +}; + +template <class T> +struct TRefCountedHelper<T, true> +{ + Y_FORCE_INLINE static const TRefCounter* GetRefCounter(const T* obj) + { + return obj; + } + + Y_FORCE_INLINE static void Destroy(const TRefCountedBase* obj) + { + const_cast<TRefCountedBase*>(obj)->DestroyRefCounted(); + } + + Y_FORCE_INLINE static void Deallocate(const TRefCountedBase* obj) + { + auto* ptr = reinterpret_cast<void**>(const_cast<TRefCountedBase*>(obj)); + auto [offset, ptrToDeleter] = UnpackPointer<void(void*, uint16_t)>(*ptr); + + // The most derived type is erased here. So we cannot call TMemoryReleaser with derived type. + ptrToDeleter(reinterpret_cast<char*>(ptr) - offset, offset); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> Y_FORCE_INLINE const TRefCounter* GetRefCounter(const T* obj) -{ - return TRefCountedHelper<T>::GetRefCounter(obj); +{ + return TRefCountedHelper<T>::GetRefCounter(obj); } -template <class T> +template <class T> Y_FORCE_INLINE void DestroyRefCounted(const T* obj) { - TRefCountedHelper<T>::Destroy(obj); + TRefCountedHelper<T>::Destroy(obj); } -template <class T> +template <class T> Y_FORCE_INLINE void DeallocateRefCounted(const T* obj) +{ + TRefCountedHelper<T>::Deallocate(obj); +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +Y_FORCE_INLINE void Ref(T* obj) +{ + GetRefCounter(obj)->Ref(); +} + +template <class T> +Y_FORCE_INLINE void Unref(T* obj) +{ + if (GetRefCounter(obj)->Unref()) { + DestroyRefCounted(obj); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +Y_FORCE_INLINE void TRefCounted::Unref() const { - TRefCountedHelper<T>::Deallocate(obj); -} - -//////////////////////////////////////////////////////////////////////////////// - -template <class T> -Y_FORCE_INLINE void Ref(T* obj) -{ - GetRefCounter(obj)->Ref(); + ::NYT::Unref(this); } +Y_FORCE_INLINE void TRefCounted::WeakUnref() const +{ + if (TRefCounter::WeakUnref()) { + DeallocateRefCounted(this); + } +} + + template <class T> -Y_FORCE_INLINE void Unref(T* obj) +void TRefCounted::DestroyRefCountedImpl(T* ptr) { - if (GetRefCounter(obj)->Unref()) { - DestroyRefCounted(obj); - } -} - -//////////////////////////////////////////////////////////////////////////////// - -Y_FORCE_INLINE void TRefCounted::Unref() const -{ - ::NYT::Unref(this); -} - -Y_FORCE_INLINE void TRefCounted::WeakUnref() const -{ - if (TRefCounter::WeakUnref()) { - DeallocateRefCounted(this); - } -} - - -template <class T> -void TRefCounted::DestroyRefCountedImpl(T* ptr) -{ - // No standard way to statically calculate the base offset even if T is final. - // static_cast<TFinalDerived*>(virtualBasePtr) does not work. - - auto* basePtr = static_cast<TRefCountedBase*>(ptr); - auto offset = reinterpret_cast<uintptr_t>(basePtr) - reinterpret_cast<uintptr_t>(ptr); - auto* refCounter = GetRefCounter(ptr); - - // No virtual call when T is final. - ptr->~T(); - - // Fast path. Weak refs cannot appear if there are neither strong nor weak refs. - if (refCounter->GetWeakRefCount() == 1) { - TMemoryReleaser<T>::Do(ptr, offset); - return; - } - - YT_ASSERT(offset < std::numeric_limits<uint16_t>::max()); - - auto* vTablePtr = reinterpret_cast<char**>(basePtr); - *vTablePtr = PackPointer(&TMemoryReleaser<T>::Do, offset); - - if (refCounter->WeakUnref()) { - TMemoryReleaser<T>::Do(ptr, offset); - } + // No standard way to statically calculate the base offset even if T is final. + // static_cast<TFinalDerived*>(virtualBasePtr) does not work. + + auto* basePtr = static_cast<TRefCountedBase*>(ptr); + auto offset = reinterpret_cast<uintptr_t>(basePtr) - reinterpret_cast<uintptr_t>(ptr); + auto* refCounter = GetRefCounter(ptr); + + // No virtual call when T is final. + ptr->~T(); + + // Fast path. Weak refs cannot appear if there are neither strong nor weak refs. + if (refCounter->GetWeakRefCount() == 1) { + TMemoryReleaser<T>::Do(ptr, offset); + return; + } + + YT_ASSERT(offset < std::numeric_limits<uint16_t>::max()); + + auto* vTablePtr = reinterpret_cast<char**>(basePtr); + *vTablePtr = PackPointer(&TMemoryReleaser<T>::Do, offset); + + if (refCounter->WeakUnref()) { + TMemoryReleaser<T>::Do(ptr, offset); + } } //////////////////////////////////////////////////////////////////////////////// diff --git a/library/cpp/yt/memory/ref_counted.h b/library/cpp/yt/memory/ref_counted.h index b683615b83..cf03aedc1f 100644 --- a/library/cpp/yt/memory/ref_counted.h +++ b/library/cpp/yt/memory/ref_counted.h @@ -5,23 +5,23 @@ #include <library/cpp/yt/assert/assert.h> #include <library/cpp/ytalloc/api/ytalloc.h> - + #include <atomic> namespace NYT { //////////////////////////////////////////////////////////////////////////////// -//! A technical base class for ref-counted objects and promise states. +//! A technical base class for ref-counted objects and promise states. class TRefCountedBase { public: TRefCountedBase() = default; - - // Make destructor protected + + // Make destructor protected virtual ~TRefCountedBase() noexcept = default; - virtual void DestroyRefCounted() = 0; + virtual void DestroyRefCounted() = 0; private: TRefCountedBase(const TRefCountedBase&) = delete; @@ -33,156 +33,156 @@ private: //////////////////////////////////////////////////////////////////////////////// -template <class T, class = void> -struct TFreeMemory +template <class T, class = void> +struct TFreeMemory { - static void Do(void* ptr) - { - NYTAlloc::FreeNonNull(ptr); - } -}; + static void Do(void* ptr) + { + NYTAlloc::FreeNonNull(ptr); + } +}; -template <class T> +template <class T> struct TFreeMemory<T, std::void_t<typename T::TAllocator>> -{ - static void Do(void* ptr) - { - using TAllocator = typename T::TAllocator; - TAllocator::Free(ptr); - } -}; +{ + static void Do(void* ptr) + { + using TAllocator = typename T::TAllocator; + TAllocator::Free(ptr); + } +}; -//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// class TRefCounter -{ +{ public: - //! Returns current number of strong references to the object. + //! Returns current number of strong references to the object. /*! * Note that you should never ever use this method in production code. * This method is mainly for debugging purposes. */ int GetRefCount() const noexcept; - //! Increments the strong reference counter. - void Ref() const noexcept; - - //! Increments the strong reference counter if it is not null. - bool TryRef() const noexcept; - - //! Decrements the strong reference counter. - bool Unref() const; - - //! Returns current number of weak references to the object. - int GetWeakRefCount() const noexcept; - - //! Increments the weak reference counter. - void WeakRef() const noexcept; - - //! Decrements the weak reference counter. - bool WeakUnref() const; - -private: - mutable std::atomic<int> StrongCount_ = 1; - mutable std::atomic<int> WeakCount_ = 1; -}; - -//////////////////////////////////////////////////////////////////////////////// - -template <class T> + //! Increments the strong reference counter. + void Ref() const noexcept; + + //! Increments the strong reference counter if it is not null. + bool TryRef() const noexcept; + + //! Decrements the strong reference counter. + bool Unref() const; + + //! Returns current number of weak references to the object. + int GetWeakRefCount() const noexcept; + + //! Increments the weak reference counter. + void WeakRef() const noexcept; + + //! Decrements the weak reference counter. + bool WeakUnref() const; + +private: + mutable std::atomic<int> StrongCount_ = 1; + mutable std::atomic<int> WeakCount_ = 1; +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> const TRefCounter* GetRefCounter(const T* obj); - -template <class T> + +template <class T> void DestroyRefCounted(const T* obj); - -template <class T> + +template <class T> void DeallocateRefCounted(const T* obj); - -//////////////////////////////////////////////////////////////////////////////// - -// API - -template <class T> -void Ref(T* obj); - -template <class T> -void Unref(T* obj); - -//////////////////////////////////////////////////////////////////////////////// - -struct TRefCounted - : public TRefCountedBase - , public TRefCounter -{ - void Unref() const; - - void WeakUnref() const; - - template <class T> - static void DestroyRefCountedImpl(T* ptr); -}; - -//////////////////////////////////////////////////////////////////////////////// - -// Forward declaration. -template <class T> -class TIntrusivePtr; - -using TRefCountedPtr = TIntrusivePtr<TRefCounted>; - -// A bunch of helpful macros that enable working with intrusive pointers to incomplete types. -/* - * Typically when you have a forward-declared type |T| and an instance - * of |TIntrusivePtr<T>| you need the complete definition of |T| to work with - * the pointer even if you're not actually using the members of |T|. - * E.g. the dtor of |TIntrusivePtr<T>|, should you ever need it, must be able - * to unref an instance of |T| and eventually destroy it. - * This may force #inclusion of way more headers than really seems necessary. - * - * |DECLARE_REFCOUNTED_STRUCT|, |DECLARE_REFCOUNTED_CLASS|, and |DEFINE_REFCOUNTED_TYPE| - * alleviate this issue by forcing TIntrusivePtr to work with the free-standing overloads - * of |Ref| and |Unref| instead of their template version. - * These overloads are declared together with the forward declaration of |T| and - * are subsequently defined afterwards. - */ - -#define DECLARE_REFCOUNTED_TYPE(type) \ + +//////////////////////////////////////////////////////////////////////////////// + +// API + +template <class T> +void Ref(T* obj); + +template <class T> +void Unref(T* obj); + +//////////////////////////////////////////////////////////////////////////////// + +struct TRefCounted + : public TRefCountedBase + , public TRefCounter +{ + void Unref() const; + + void WeakUnref() const; + + template <class T> + static void DestroyRefCountedImpl(T* ptr); +}; + +//////////////////////////////////////////////////////////////////////////////// + +// Forward declaration. +template <class T> +class TIntrusivePtr; + +using TRefCountedPtr = TIntrusivePtr<TRefCounted>; + +// A bunch of helpful macros that enable working with intrusive pointers to incomplete types. +/* + * Typically when you have a forward-declared type |T| and an instance + * of |TIntrusivePtr<T>| you need the complete definition of |T| to work with + * the pointer even if you're not actually using the members of |T|. + * E.g. the dtor of |TIntrusivePtr<T>|, should you ever need it, must be able + * to unref an instance of |T| and eventually destroy it. + * This may force #inclusion of way more headers than really seems necessary. + * + * |DECLARE_REFCOUNTED_STRUCT|, |DECLARE_REFCOUNTED_CLASS|, and |DEFINE_REFCOUNTED_TYPE| + * alleviate this issue by forcing TIntrusivePtr to work with the free-standing overloads + * of |Ref| and |Unref| instead of their template version. + * These overloads are declared together with the forward declaration of |T| and + * are subsequently defined afterwards. + */ + +#define DECLARE_REFCOUNTED_TYPE(type) \ using type ## Ptr = ::NYT::TIntrusivePtr<type>; \ - \ + \ [[maybe_unused]] ATTRIBUTE_USED const ::NYT::TRefCounter* GetRefCounter(const type* obj); \ [[maybe_unused]] ATTRIBUTE_USED void DestroyRefCounted(const type* obj); \ [[maybe_unused]] ATTRIBUTE_USED void DeallocateRefCounted(const type* obj); - -//! Forward-declares a class type, defines an intrusive pointer for it, and finally -//! declares Ref/Unref overloads. Use this macro in |public.h|-like files. -#define DECLARE_REFCOUNTED_CLASS(type) \ - class type; \ - DECLARE_REFCOUNTED_TYPE(type) - -//! Forward-declares a struct type, defines an intrusive pointer for it, and finally -//! declares Ref/Unref overloads. Use this macro in |public.h|-like files. -#define DECLARE_REFCOUNTED_STRUCT(type) \ - struct type; \ - DECLARE_REFCOUNTED_TYPE(type) - -//! Provides implementations for Ref/Unref overloads. Use this macro right -//! after the type's full definition. -#define DEFINE_REFCOUNTED_TYPE(type) \ + +//! Forward-declares a class type, defines an intrusive pointer for it, and finally +//! declares Ref/Unref overloads. Use this macro in |public.h|-like files. +#define DECLARE_REFCOUNTED_CLASS(type) \ + class type; \ + DECLARE_REFCOUNTED_TYPE(type) + +//! Forward-declares a struct type, defines an intrusive pointer for it, and finally +//! declares Ref/Unref overloads. Use this macro in |public.h|-like files. +#define DECLARE_REFCOUNTED_STRUCT(type) \ + struct type; \ + DECLARE_REFCOUNTED_TYPE(type) + +//! Provides implementations for Ref/Unref overloads. Use this macro right +//! after the type's full definition. +#define DEFINE_REFCOUNTED_TYPE(type) \ [[maybe_unused]] ATTRIBUTE_USED Y_FORCE_INLINE const ::NYT::TRefCounter* GetRefCounter(const type* obj) \ - { \ - return ::NYT::TRefCountedHelper<type>::GetRefCounter(obj); \ - } \ + { \ + return ::NYT::TRefCountedHelper<type>::GetRefCounter(obj); \ + } \ [[maybe_unused]] ATTRIBUTE_USED Y_FORCE_INLINE void DestroyRefCounted(const type* obj) \ - { \ - ::NYT::TRefCountedHelper<type>::Destroy(obj); \ - } \ + { \ + ::NYT::TRefCountedHelper<type>::Destroy(obj); \ + } \ [[maybe_unused]] ATTRIBUTE_USED Y_FORCE_INLINE void DeallocateRefCounted(const type* obj) \ - { \ - ::NYT::TRefCountedHelper<type>::Deallocate(obj); \ - } - -//////////////////////////////////////////////////////////////////////////////// - + { \ + ::NYT::TRefCountedHelper<type>::Deallocate(obj); \ + } + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT #define REF_COUNTED_INL_H_ diff --git a/library/cpp/yt/memory/ref_tracked-inl.h b/library/cpp/yt/memory/ref_tracked-inl.h index 4bde72881d..438bd870a9 100644 --- a/library/cpp/yt/memory/ref_tracked-inl.h +++ b/library/cpp/yt/memory/ref_tracked-inl.h @@ -1,49 +1,49 @@ -#ifndef REF_TRACKED_INL_H_ -#error "Direct inclusion of this file is not allowed, include ref_tracked.h" -// For the sake of sane code completion. -#include "ref_tracked.h" -#endif - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -template <class T> -TRefCountedTypeKey GetRefCountedTypeKey() -{ - return &typeid(T); -} - -template <class T> -Y_FORCE_INLINE TRefCountedTypeCookie GetRefCountedTypeCookie() -{ - static std::atomic<TRefCountedTypeCookie> cookie{NullRefCountedTypeCookie}; - auto cookieValue = cookie.load(std::memory_order_relaxed); - if (Y_UNLIKELY(cookieValue == NullRefCountedTypeCookie)) { - cookieValue = TRefCountedTrackerFacade::GetCookie( - GetRefCountedTypeKey<T>(), - sizeof(T), - NYT::TSourceLocation()); - cookie.store(cookieValue, std::memory_order_relaxed); - } - return cookieValue; -} - -template <class T, class TTag, int Counter> -Y_FORCE_INLINE TRefCountedTypeCookie GetRefCountedTypeCookieWithLocation(const TSourceLocation& location) -{ - static std::atomic<TRefCountedTypeCookie> cookie{NullRefCountedTypeCookie}; - auto cookieValue = cookie.load(std::memory_order_relaxed); - if (Y_UNLIKELY(cookieValue == NullRefCountedTypeCookie)) { - cookieValue = TRefCountedTrackerFacade::GetCookie( - GetRefCountedTypeKey<T>(), - sizeof(T), - location); - cookie.store(cookieValue, std::memory_order_relaxed); - } - return cookieValue; -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT +#ifndef REF_TRACKED_INL_H_ +#error "Direct inclusion of this file is not allowed, include ref_tracked.h" +// For the sake of sane code completion. +#include "ref_tracked.h" +#endif + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +TRefCountedTypeKey GetRefCountedTypeKey() +{ + return &typeid(T); +} + +template <class T> +Y_FORCE_INLINE TRefCountedTypeCookie GetRefCountedTypeCookie() +{ + static std::atomic<TRefCountedTypeCookie> cookie{NullRefCountedTypeCookie}; + auto cookieValue = cookie.load(std::memory_order_relaxed); + if (Y_UNLIKELY(cookieValue == NullRefCountedTypeCookie)) { + cookieValue = TRefCountedTrackerFacade::GetCookie( + GetRefCountedTypeKey<T>(), + sizeof(T), + NYT::TSourceLocation()); + cookie.store(cookieValue, std::memory_order_relaxed); + } + return cookieValue; +} + +template <class T, class TTag, int Counter> +Y_FORCE_INLINE TRefCountedTypeCookie GetRefCountedTypeCookieWithLocation(const TSourceLocation& location) +{ + static std::atomic<TRefCountedTypeCookie> cookie{NullRefCountedTypeCookie}; + auto cookieValue = cookie.load(std::memory_order_relaxed); + if (Y_UNLIKELY(cookieValue == NullRefCountedTypeCookie)) { + cookieValue = TRefCountedTrackerFacade::GetCookie( + GetRefCountedTypeKey<T>(), + sizeof(T), + location); + cookie.store(cookieValue, std::memory_order_relaxed); + } + return cookieValue; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yt/memory/ref_tracked.cpp b/library/cpp/yt/memory/ref_tracked.cpp index 4dafbc0849..5eaa17bd42 100644 --- a/library/cpp/yt/memory/ref_tracked.cpp +++ b/library/cpp/yt/memory/ref_tracked.cpp @@ -1,38 +1,38 @@ -#include "ref_tracked.h" - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -Y_WEAK TRefCountedTypeCookie TRefCountedTrackerFacade::GetCookie( - TRefCountedTypeKey /*typeKey*/, - size_t /*instanceSize*/, - const TSourceLocation& /*location*/) -{ - return NullRefCountedTypeCookie; -} - -Y_WEAK void TRefCountedTrackerFacade::AllocateInstance(TRefCountedTypeCookie /*cookie*/) -{ } - -Y_WEAK void TRefCountedTrackerFacade::FreeInstance(TRefCountedTypeCookie /*cookie*/) -{ } - -Y_WEAK void TRefCountedTrackerFacade::AllocateTagInstance(TRefCountedTypeCookie /*cookie*/) -{ } - -Y_WEAK void TRefCountedTrackerFacade::FreeTagInstance(TRefCountedTypeCookie /*cookie*/) -{ } - -Y_WEAK void TRefCountedTrackerFacade::AllocateSpace(TRefCountedTypeCookie /*cookie*/, size_t /*size*/) -{ } - -Y_WEAK void TRefCountedTrackerFacade::FreeSpace(TRefCountedTypeCookie /*cookie*/, size_t /*size*/) -{ } - -Y_WEAK void TRefCountedTrackerFacade::Dump() -{ } - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT +#include "ref_tracked.h" + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +Y_WEAK TRefCountedTypeCookie TRefCountedTrackerFacade::GetCookie( + TRefCountedTypeKey /*typeKey*/, + size_t /*instanceSize*/, + const TSourceLocation& /*location*/) +{ + return NullRefCountedTypeCookie; +} + +Y_WEAK void TRefCountedTrackerFacade::AllocateInstance(TRefCountedTypeCookie /*cookie*/) +{ } + +Y_WEAK void TRefCountedTrackerFacade::FreeInstance(TRefCountedTypeCookie /*cookie*/) +{ } + +Y_WEAK void TRefCountedTrackerFacade::AllocateTagInstance(TRefCountedTypeCookie /*cookie*/) +{ } + +Y_WEAK void TRefCountedTrackerFacade::FreeTagInstance(TRefCountedTypeCookie /*cookie*/) +{ } + +Y_WEAK void TRefCountedTrackerFacade::AllocateSpace(TRefCountedTypeCookie /*cookie*/, size_t /*size*/) +{ } + +Y_WEAK void TRefCountedTrackerFacade::FreeSpace(TRefCountedTypeCookie /*cookie*/, size_t /*size*/) +{ } + +Y_WEAK void TRefCountedTrackerFacade::Dump() +{ } + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yt/memory/ref_tracked.h b/library/cpp/yt/memory/ref_tracked.h index 75c1eb5985..c43940ea3e 100644 --- a/library/cpp/yt/memory/ref_tracked.h +++ b/library/cpp/yt/memory/ref_tracked.h @@ -3,67 +3,67 @@ #include <library/cpp/yt/misc/port.h> #include <library/cpp/yt/misc/source_location.h> -#include <util/system/defaults.h> - -#include <atomic> -#include <typeinfo> - +#include <util/system/defaults.h> + +#include <atomic> +#include <typeinfo> + namespace NYT { //////////////////////////////////////////////////////////////////////////////// -using TRefCountedTypeCookie = int; -const int NullRefCountedTypeCookie = -1; - -using TRefCountedTypeKey = const void*; - -//////////////////////////////////////////////////////////////////////////////// - -// Used to avoid including heavy ref_counted_tracker.h -class TRefCountedTrackerFacade -{ -public: - static TRefCountedTypeCookie GetCookie( - TRefCountedTypeKey typeKey, - size_t instanceSize, - const NYT::TSourceLocation& location); - - static void AllocateInstance(TRefCountedTypeCookie cookie); - static void FreeInstance(TRefCountedTypeCookie cookie); - - static void AllocateTagInstance(TRefCountedTypeCookie cookie); - static void FreeTagInstance(TRefCountedTypeCookie cookie); - - static void AllocateSpace(TRefCountedTypeCookie cookie, size_t size); - static void FreeSpace(TRefCountedTypeCookie cookie, size_t size); - - // Typically invoked from GDB console. - // Dumps the ref-counted statistics sorted by "bytes alive". - static void Dump(); -}; - -//////////////////////////////////////////////////////////////////////////////// - -namespace { - -//! A per-translation unit tag type. -struct TCurrentTranslationUnitTag -{ }; - -} // namespace - -template <class T> -TRefCountedTypeKey GetRefCountedTypeKey(); - -template <class T> -TRefCountedTypeCookie GetRefCountedTypeCookie(); - -template <class T, class TTag, int Counter> -TRefCountedTypeCookie GetRefCountedTypeCookieWithLocation( - const TSourceLocation& location); - -//////////////////////////////////////////////////////////////////////////////// - +using TRefCountedTypeCookie = int; +const int NullRefCountedTypeCookie = -1; + +using TRefCountedTypeKey = const void*; + +//////////////////////////////////////////////////////////////////////////////// + +// Used to avoid including heavy ref_counted_tracker.h +class TRefCountedTrackerFacade +{ +public: + static TRefCountedTypeCookie GetCookie( + TRefCountedTypeKey typeKey, + size_t instanceSize, + const NYT::TSourceLocation& location); + + static void AllocateInstance(TRefCountedTypeCookie cookie); + static void FreeInstance(TRefCountedTypeCookie cookie); + + static void AllocateTagInstance(TRefCountedTypeCookie cookie); + static void FreeTagInstance(TRefCountedTypeCookie cookie); + + static void AllocateSpace(TRefCountedTypeCookie cookie, size_t size); + static void FreeSpace(TRefCountedTypeCookie cookie, size_t size); + + // Typically invoked from GDB console. + // Dumps the ref-counted statistics sorted by "bytes alive". + static void Dump(); +}; + +//////////////////////////////////////////////////////////////////////////////// + +namespace { + +//! A per-translation unit tag type. +struct TCurrentTranslationUnitTag +{ }; + +} // namespace + +template <class T> +TRefCountedTypeKey GetRefCountedTypeKey(); + +template <class T> +TRefCountedTypeCookie GetRefCountedTypeCookie(); + +template <class T, class TTag, int Counter> +TRefCountedTypeCookie GetRefCountedTypeCookieWithLocation( + const TSourceLocation& location); + +//////////////////////////////////////////////////////////////////////////////// + //! A lightweight mix-in that integrates any class into TRefCountedTracker statistics. /*! * |T| must be the actual derived type. @@ -75,7 +75,7 @@ template <class T> class TRefTracked { public: -#ifdef YT_ENABLE_REF_COUNTED_TRACKING +#ifdef YT_ENABLE_REF_COUNTED_TRACKING TRefTracked() { auto cookie = GetRefCountedTypeCookie<T>(); @@ -98,14 +98,14 @@ public: { auto cookie = GetRefCountedTypeCookie<T>(); TRefCountedTrackerFacade::FreeInstance(cookie); - } + } #endif }; //////////////////////////////////////////////////////////////////////////////// } // namespace NYT - -#define REF_TRACKED_INL_H_ -#include "ref_tracked-inl.h" -#undef REF_TRACKED_INL_H_ + +#define REF_TRACKED_INL_H_ +#include "ref_tracked-inl.h" +#undef REF_TRACKED_INL_H_ diff --git a/library/cpp/yt/memory/shared_range.h b/library/cpp/yt/memory/shared_range.h index 9841d7a0df..b50a224bc1 100644 --- a/library/cpp/yt/memory/shared_range.h +++ b/library/cpp/yt/memory/shared_range.h @@ -19,7 +19,7 @@ class TSharedRange : public TRange<T> { public: - using THolderPtr = TRefCountedPtr; + using THolderPtr = TRefCountedPtr; //! Constructs a null TSharedRange. TSharedRange() @@ -104,7 +104,7 @@ protected: //! Constructs a combined holder instance by taking ownership of a given list of holders. template <class... THolders> -TRefCountedPtr MakeCompositeHolder(THolders&&... holders) +TRefCountedPtr MakeCompositeHolder(THolders&&... holders) { struct THolder : public TRefCounted @@ -185,7 +185,7 @@ class TSharedMutableRange : public TMutableRange<T> { public: - using THolderPtr = TRefCountedPtr; + using THolderPtr = TRefCountedPtr; //! Constructs a null TSharedMutableRange. TSharedMutableRange() diff --git a/library/cpp/yt/memory/unittests/intrusive_ptr_ut.cpp b/library/cpp/yt/memory/unittests/intrusive_ptr_ut.cpp index 622bed0eb0..5b19a9add8 100644 --- a/library/cpp/yt/memory/unittests/intrusive_ptr_ut.cpp +++ b/library/cpp/yt/memory/unittests/intrusive_ptr_ut.cpp @@ -1,4 +1,4 @@ -#include <library/cpp/testing/gtest/gtest.h> +#include <library/cpp/testing/gtest/gtest.h> #include <library/cpp/yt/memory/new.h> #include <library/cpp/yt/memory/ref_counted.h> @@ -68,9 +68,9 @@ MATCHER_P3(HasRefCounts, increments, decrements, zeros, void PrintTo(const TIntricateObject& arg, ::std::ostream* os) { - *os << arg.Increments << " increments, " - << arg.Decrements << " decrements and " - << arg.Zeros << " times vanished"; + *os << arg.Increments << " increments, " + << arg.Decrements << " decrements and " + << arg.Zeros << " times vanished"; } // This is an object which creates intrusive pointers to the self @@ -480,82 +480,82 @@ TEST(TIntrusivePtrTest, ObjectAlignment) EXPECT_TRUE(reinterpret_cast<uintptr_t>(bar.Get()) % 64 == 0); } -TEST(TIntrusivePtrTest, InitStruct) -{ - struct TObj1 final - { - const int A; - const int B; - }; - - New<TObj1>(1, 2); - - struct TExplicitObj final - { - explicit TExplicitObj(int a = 0) - : A(a) - { } - - const int A; - }; - - New<TExplicitObj>(); - New<TExplicitObj>(1); - - struct TObj2 final - { - TObj2(i64 a = 0) - : A(a) - { } - - const i64 A; - }; - - New<TObj2>(123); - - struct TObj3 final - { - TObj3(ui64 a = 0) - : A(a) - { } - - const ui64 A; - }; - - New<TObj3>(123); - - struct TObj4 final - { - TObj4(int a, ui64 b = 0) - : A(a) - , B(b) - { } - - int A; - const ui64 B; - }; - - New<TObj4>(123); - New<TObj4>(123, 123); - - struct TObj5 final - { - TExplicitObj E; - int B; - }; - - New<TObj5>(); - - struct TObj6 final - { - TObj2 O; - int B; - }; - - New<TObj6>(); - New<TObj6>(1, 2); -} - +TEST(TIntrusivePtrTest, InitStruct) +{ + struct TObj1 final + { + const int A; + const int B; + }; + + New<TObj1>(1, 2); + + struct TExplicitObj final + { + explicit TExplicitObj(int a = 0) + : A(a) + { } + + const int A; + }; + + New<TExplicitObj>(); + New<TExplicitObj>(1); + + struct TObj2 final + { + TObj2(i64 a = 0) + : A(a) + { } + + const i64 A; + }; + + New<TObj2>(123); + + struct TObj3 final + { + TObj3(ui64 a = 0) + : A(a) + { } + + const ui64 A; + }; + + New<TObj3>(123); + + struct TObj4 final + { + TObj4(int a, ui64 b = 0) + : A(a) + , B(b) + { } + + int A; + const ui64 B; + }; + + New<TObj4>(123); + New<TObj4>(123, 123); + + struct TObj5 final + { + TExplicitObj E; + int B; + }; + + New<TObj5>(); + + struct TObj6 final + { + TObj2 O; + int B; + }; + + New<TObj6>(); + New<TObj6>(1, 2); +} + //////////////////////////////////////////////////////////////////////////////// } // namespace diff --git a/library/cpp/yt/memory/unittests/weak_ptr_ut.cpp b/library/cpp/yt/memory/unittests/weak_ptr_ut.cpp index 180c16b5ca..c00451badd 100644 --- a/library/cpp/yt/memory/unittests/weak_ptr_ut.cpp +++ b/library/cpp/yt/memory/unittests/weak_ptr_ut.cpp @@ -1,9 +1,9 @@ -#include <library/cpp/testing/gtest/gtest.h> +#include <library/cpp/testing/gtest/gtest.h> #include <library/cpp/yt/memory/new.h> #include <library/cpp/yt/memory/weak_ptr.h> -#include <array> +#include <array> namespace NYT { namespace { @@ -91,8 +91,8 @@ MATCHER_P2(HasRefCounts, strongRefs, weakRefs, template <class T> void PrintExtrinsicRefCounted(const T& arg, ::std::ostream* os) { - *os << arg.GetRefCount() << " strong and " - << arg.GetWeakRefCount() << " weak references"; + *os << arg.GetRefCount() << " strong and " + << arg.GetWeakRefCount() << " weak references"; } void PrintTo(const TIntricateObject& arg, ::std::ostream* os) @@ -353,32 +353,32 @@ TEST_F(TWeakPtrTest, VirtualBase) ptr.Reset(); } -#if 0 -class TSlowlyDyingObject - : public TRefCounted -{ -public: - TSlowlyDyingObject() - { - ++ConstructorShadowState; - } - - virtual ~TSlowlyDyingObject() - { - ++DestructorShadowState; - DeathEvent->Wait(); - ++DestructorShadowState; - } -}; - -void PrintTo(const TSlowlyDyingObject& arg, ::std::ostream* os) -{ - PrintExtrinsicRefCounted(arg, os); -} - -typedef TIntrusivePtr<TSlowlyDyingObject> TSlowlyDyingObjectPtr; -typedef TWeakPtr<TSlowlyDyingObject> TSlowlyDyingObjectWkPtr; - +#if 0 +class TSlowlyDyingObject + : public TRefCounted +{ +public: + TSlowlyDyingObject() + { + ++ConstructorShadowState; + } + + virtual ~TSlowlyDyingObject() + { + ++DestructorShadowState; + DeathEvent->Wait(); + ++DestructorShadowState; + } +}; + +void PrintTo(const TSlowlyDyingObject& arg, ::std::ostream* os) +{ + PrintExtrinsicRefCounted(arg, os); +} + +typedef TIntrusivePtr<TSlowlyDyingObject> TSlowlyDyingObjectPtr; +typedef TWeakPtr<TSlowlyDyingObject> TSlowlyDyingObjectWkPtr; + static void* AsynchronousDeleter(void* param) { TSlowlyDyingObjectPtr* indirectObject = @@ -388,11 +388,11 @@ static void* AsynchronousDeleter(void* param) } std::unique_ptr<NThreading::TEvent> DeathEvent; - + TEST_F(TWeakPtrTest, DISABLED_AcquisionOfSlowlyDyingObject) { DeathEvent.reset(new NThreading::TEvent()); - + TSlowlyDyingObjectPtr object = New<TSlowlyDyingObject>(); TSlowlyDyingObjectWkPtr ptr(object); @@ -425,8 +425,8 @@ TEST_F(TWeakPtrTest, DISABLED_AcquisionOfSlowlyDyingObject) EXPECT_EQ(TSlowlyDyingObjectPtr(), ptr.Lock()); } -#endif - +#endif + //////////////////////////////////////////////////////////////////////////////// } // namespace diff --git a/library/cpp/yt/memory/unittests/ya.make b/library/cpp/yt/memory/unittests/ya.make index f09ad7d0c9..4b86574609 100644 --- a/library/cpp/yt/memory/unittests/ya.make +++ b/library/cpp/yt/memory/unittests/ya.make @@ -1,19 +1,19 @@ -GTEST(unittester-library-memory) - -OWNER(g:yt) - +GTEST(unittester-library-memory) + +OWNER(g:yt) + IF (NOT OS_WINDOWS) ALLOCATOR(YT) ENDIF() - -SRCS( - intrusive_ptr_ut.cpp - weak_ptr_ut.cpp -) - -PEERDIR( - library/cpp/testing/gtest + +SRCS( + intrusive_ptr_ut.cpp + weak_ptr_ut.cpp +) + +PEERDIR( + library/cpp/testing/gtest library/cpp/yt/memory -) - -END() +) + +END() diff --git a/library/cpp/yt/memory/weak_ptr.h b/library/cpp/yt/memory/weak_ptr.h index 25a242bb8a..7a789f3b0b 100644 --- a/library/cpp/yt/memory/weak_ptr.h +++ b/library/cpp/yt/memory/weak_ptr.h @@ -1,6 +1,6 @@ #pragma once -#include "ref_counted.h" +#include "ref_counted.h" #include <util/generic/hash.h> @@ -8,7 +8,7 @@ namespace NYT { //////////////////////////////////////////////////////////////////////////////// -template <class T> +template <class T> class TWeakPtr { public: @@ -28,84 +28,84 @@ public: explicit TWeakPtr(T* p) noexcept : T_(p) { - -#if defined(_tsan_enabled_) + +#if defined(_tsan_enabled_) if (T_) { - RefCounter_ = GetRefCounter(T_); + RefCounter_ = GetRefCounter(T_); } -#endif - AcquireRef(); +#endif + AcquireRef(); } //! Constructor from a strong reference. - TWeakPtr(const TIntrusivePtr<T>& ptr) noexcept - : TWeakPtr(ptr.Get()) - { } + TWeakPtr(const TIntrusivePtr<T>& ptr) noexcept + : TWeakPtr(ptr.Get()) + { } - //! Constructor from a strong reference with an upcast. + //! Constructor from a strong reference with an upcast. template <class U, class = typename std::enable_if_t<std::is_convertible_v<U*, T*>>> - TWeakPtr(const TIntrusivePtr<U>& ptr) noexcept - : TWeakPtr(ptr.Get()) - { - static_assert( - std::is_base_of_v<TRefCountedBase, T>, - "Cast allowed only for types derived from TRefCountedBase"); - } - + TWeakPtr(const TIntrusivePtr<U>& ptr) noexcept + : TWeakPtr(ptr.Get()) + { + static_assert( + std::is_base_of_v<TRefCountedBase, T>, + "Cast allowed only for types derived from TRefCountedBase"); + } + //! Copy constructor. - TWeakPtr(const TWeakPtr& other) noexcept - : TWeakPtr(other.T_) - { } + TWeakPtr(const TWeakPtr& other) noexcept + : TWeakPtr(other.T_) + { } //! Copy constructor with an upcast. template <class U, class = typename std::enable_if_t<std::is_convertible_v<U*, T*>>> - TWeakPtr(const TWeakPtr<U>& other) noexcept - : TWeakPtr(other.Lock()) + TWeakPtr(const TWeakPtr<U>& other) noexcept + : TWeakPtr(other.Lock()) { - static_assert( - std::is_base_of_v<TRefCountedBase, T>, - "Cast allowed only for types derived from TRefCountedBase"); + static_assert( + std::is_base_of_v<TRefCountedBase, T>, + "Cast allowed only for types derived from TRefCountedBase"); } //! Move constructor. - TWeakPtr(TWeakPtr&& other) noexcept + TWeakPtr(TWeakPtr&& other) noexcept { - other.Swap(*this); + other.Swap(*this); } //! Move constructor with an upcast. template <class U, class = typename std::enable_if_t<std::is_convertible_v<U*, T*>>> - TWeakPtr(TWeakPtr<U>&& other) noexcept + TWeakPtr(TWeakPtr<U>&& other) noexcept { - static_assert( - std::is_base_of_v<TRefCountedBase, T>, - "Cast allowed only for types derived from TRefCountedBase"); + static_assert( + std::is_base_of_v<TRefCountedBase, T>, + "Cast allowed only for types derived from TRefCountedBase"); TIntrusivePtr<U> strongOther = other.Lock(); if (strongOther) { T_ = other.T_; - other.T_ = nullptr; + other.T_ = nullptr; -#if defined(_tsan_enabled_) - RefCounter_ = other.RefCounter_; - other.RefCounter_ = nullptr; -#endif +#if defined(_tsan_enabled_) + RefCounter_ = other.RefCounter_; + other.RefCounter_ = nullptr; +#endif } } //! Destructor. ~TWeakPtr() { - ReleaseRef(); + ReleaseRef(); } //! Assignment operator from a strong reference. template <class U> - TWeakPtr& operator=(const TIntrusivePtr<U>& ptr) noexcept + TWeakPtr& operator=(const TIntrusivePtr<U>& ptr) noexcept { static_assert( std::is_convertible_v<U*, T*>, - "U* must be convertible to T*"); - TWeakPtr(ptr).Swap(*this); + "U* must be convertible to T*"); + TWeakPtr(ptr).Swap(*this); return *this; } @@ -122,7 +122,7 @@ public: { static_assert( std::is_convertible_v<U*, T*>, - "U* must be convertible to T*"); + "U* must be convertible to T*"); TWeakPtr(other).Swap(*this); return *this; } @@ -130,7 +130,7 @@ public: //! Move assignment operator. TWeakPtr& operator=(TWeakPtr&& other) noexcept { - other.Swap(*this); + other.Swap(*this); return *this; } @@ -138,9 +138,9 @@ public: template <class U> TWeakPtr& operator=(TWeakPtr<U>&& other) noexcept { - static_assert( + static_assert( std::is_convertible_v<U*, T*>, - "U* must be convertible to T*"); + "U* must be convertible to T*"); TWeakPtr(std::move(other)).Swap(*this); return *this; } @@ -159,73 +159,73 @@ public: //! Replace the pointer with a specified one. template <class U> - void Reset(const TIntrusivePtr<U>& ptr) // noexcept + void Reset(const TIntrusivePtr<U>& ptr) // noexcept { static_assert( std::is_convertible_v<U*, T*>, - "U* must be convertible to T*"); - TWeakPtr(ptr).Swap(*this); - } - - //! Swap the pointer with the other one. - void Swap(TWeakPtr& other) noexcept - { - DoSwap(T_, other.T_); -#if defined(_tsan_enabled_) - DoSwap(RefCounter_, other.RefCounter_); -#endif + "U* must be convertible to T*"); + TWeakPtr(ptr).Swap(*this); } + //! Swap the pointer with the other one. + void Swap(TWeakPtr& other) noexcept + { + DoSwap(T_, other.T_); +#if defined(_tsan_enabled_) + DoSwap(RefCounter_, other.RefCounter_); +#endif + } + //! Acquire a strong reference to the pointee and return a strong pointer. TIntrusivePtr<T> Lock() const noexcept { - return T_ && RefCounter()->TryRef() + return T_ && RefCounter()->TryRef() ? TIntrusivePtr<T>(T_, false) : TIntrusivePtr<T>(); } - bool IsExpired() const noexcept + bool IsExpired() const noexcept { - return !T_ || (RefCounter()->GetRefCount() == 0); + return !T_ || (RefCounter()->GetRefCount() == 0); } -private: - void AcquireRef() +private: + void AcquireRef() { - if (T_) { - RefCounter()->WeakRef(); - } - } - - void ReleaseRef() - { - if (T_) { - // Support incomplete type. - if (RefCounter()->WeakUnref()) { - DeallocateRefCounted(T_); - } - } + if (T_) { + RefCounter()->WeakRef(); + } } + void ReleaseRef() + { + if (T_) { + // Support incomplete type. + if (RefCounter()->WeakUnref()) { + DeallocateRefCounted(T_); + } + } + } + template <class U> friend class TWeakPtr; template <class U> friend struct ::THash; T* T_ = nullptr; -#if defined(_tsan_enabled_) - const TRefCounter* RefCounter_ = nullptr; - - const TRefCounter* RefCounter() const - { - return RefCounter_; - } -#else - const TRefCounter* RefCounter() const - { - return GetRefCounter(T_); - } -#endif +#if defined(_tsan_enabled_) + const TRefCounter* RefCounter_ = nullptr; + + const TRefCounter* RefCounter() const + { + return RefCounter_; + } +#else + const TRefCounter* RefCounter() const + { + return GetRefCounter(T_); + } +#endif }; //////////////////////////////////////////////////////////////////////////////// @@ -285,7 +285,7 @@ bool operator==(const TWeakPtr<T>& lhs, const TWeakPtr<U>& rhs) { static_assert( std::is_convertible_v<U*, T*>, - "U* must be convertible to T*"); + "U* must be convertible to T*"); return lhs.Lock().Get() == rhs.Lock().Get(); } @@ -294,7 +294,7 @@ bool operator!=(const TWeakPtr<T>& lhs, const TWeakPtr<U>& rhs) { static_assert( std::is_convertible_v<U*, T*>, - "U* must be convertible to T*"); + "U* must be convertible to T*"); return lhs.Lock().Get() != rhs.Lock().Get(); } @@ -309,6 +309,6 @@ struct THash<NYT::TWeakPtr<T>> { size_t operator () (const NYT::TWeakPtr<T>& ptr) const { - return THash<const NYT::TRefCountedBase*>()(ptr.T_); + return THash<const NYT::TRefCountedBase*>()(ptr.T_); } }; diff --git a/library/cpp/yt/memory/ya.make b/library/cpp/yt/memory/ya.make index a925c714ee..54e237ddee 100644 --- a/library/cpp/yt/memory/ya.make +++ b/library/cpp/yt/memory/ya.make @@ -1,31 +1,31 @@ -LIBRARY() - -OWNER(g:yt) - -SRCS( +LIBRARY() + +OWNER(g:yt) + +SRCS( blob.cpp ref.cpp - ref_tracked.cpp -) - -PEERDIR( + ref_tracked.cpp +) + +PEERDIR( library/cpp/yt/assert library/cpp/yt/misc - library/cpp/ytalloc/api -) - -CHECK_DEPENDENT_DIRS( - ALLOW_ONLY ALL - build - contrib - library - util + library/cpp/ytalloc/api +) + +CHECK_DEPENDENT_DIRS( + ALLOW_ONLY ALL + build + contrib + library + util library/cpp/yt/assert library/cpp/yt/misc -) - -END() - -RECURSE_FOR_TESTS( - unittests -) +) + +END() + +RECURSE_FOR_TESTS( + unittests +) diff --git a/library/cpp/yt/misc/port.h b/library/cpp/yt/misc/port.h index b24ac50995..3e5e7be617 100644 --- a/library/cpp/yt/misc/port.h +++ b/library/cpp/yt/misc/port.h @@ -59,12 +59,12 @@ #if defined(__GNUC__) || defined(__clang__) #define PER_THREAD __thread #define ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) - // Prevent GCC from throwing out functions in release builds. - #define ATTRIBUTE_USED __attribute__((used)) + // Prevent GCC from throwing out functions in release builds. + #define ATTRIBUTE_USED __attribute__((used)) #elif defined(_MSC_VER) #define PER_THREAD __declspec(thread) #define ATTRIBUTE_NO_SANITIZE_ADDRESS - #define ATTRIBUTE_USED + #define ATTRIBUTE_USED #else #error Unsupported compiler #endif diff --git a/library/cpp/yt/misc/source_location.cpp b/library/cpp/yt/misc/source_location.cpp index 8d22d43636..e65a382b1b 100644 --- a/library/cpp/yt/misc/source_location.cpp +++ b/library/cpp/yt/misc/source_location.cpp @@ -27,7 +27,7 @@ bool TSourceLocation::operator<(const TSourceLocation& other) const const char* otherFileName = other.FileName_ ? other.FileName_ : ""; int fileNameResult = strcmp(fileName, otherFileName); if (fileNameResult != 0) { - return fileNameResult < 0; + return fileNameResult < 0; } if (Line_ < other.Line_) { diff --git a/library/cpp/yt/misc/ya.make b/library/cpp/yt/misc/ya.make index bb76711ddd..9d75f7b9dc 100644 --- a/library/cpp/yt/misc/ya.make +++ b/library/cpp/yt/misc/ya.make @@ -1,26 +1,26 @@ -LIBRARY() - -OWNER(g:yt) - -SRCS( +LIBRARY() + +OWNER(g:yt) + +SRCS( guid.cpp - source_location.cpp -) - -PEERDIR( + source_location.cpp +) + +PEERDIR( library/cpp/yt/exception -) - -CHECK_DEPENDENT_DIRS( - ALLOW_ONLY ALL - build - contrib - library - util - yt/yt/library/small_containers -) - -END() +) + +CHECK_DEPENDENT_DIRS( + ALLOW_ONLY ALL + build + contrib + library + util + yt/yt/library/small_containers +) + +END() RECURSE_FOR_TESTS( unittests diff --git a/library/cpp/yt/yson_string/string.cpp b/library/cpp/yt/yson_string/string.cpp index 99d45e8616..655f770482 100644 --- a/library/cpp/yt/yson_string/string.cpp +++ b/library/cpp/yt/yson_string/string.cpp @@ -155,7 +155,7 @@ TString TYsonString::ToString() const [] (const TNullPayload&) -> TString { YT_ABORT(); }, - [&] (const TRefCountedPtr&) { + [&] (const TRefCountedPtr&) { return TString(AsStringBuf()); }, [] (const TString& payload) { diff --git a/library/cpp/yt/yson_string/string.h b/library/cpp/yt/yson_string/string.h index e13af37a6d..379b455de0 100644 --- a/library/cpp/yt/yson_string/string.h +++ b/library/cpp/yt/yson_string/string.h @@ -107,7 +107,7 @@ private: struct TNullPayload { }; - using THolder = TRefCountedPtr; + using THolder = TRefCountedPtr; std::variant<TNullPayload, THolder, TString> Payload_; |