diff options
author | thegeorg <thegeorg@yandex-team.com> | 2023-03-25 20:23:17 +0300 |
---|---|---|
committer | thegeorg <thegeorg@yandex-team.com> | 2023-03-25 20:23:17 +0300 |
commit | a50a4399c2600b05a086acdca3ba56c957d62196 (patch) | |
tree | 2cf3f6cc37ccc6bd19c33a928e07dd6c083cea72 /contrib/restricted/abseil-cpp-tstring/y_absl/container | |
parent | 76f3ccf647d9cff0e38a7989dc89480854107b78 (diff) | |
download | ydb-a50a4399c2600b05a086acdca3ba56c957d62196.tar.gz |
Update contrib/restricted/abseil-cpp-tstring to 20230125.1
Diffstat (limited to 'contrib/restricted/abseil-cpp-tstring/y_absl/container')
11 files changed, 1215 insertions, 510 deletions
diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/container/fixed_array.h b/contrib/restricted/abseil-cpp-tstring/y_absl/container/fixed_array.h index 94c69145fa..83c817448b 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/container/fixed_array.h +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/container/fixed_array.h @@ -62,11 +62,10 @@ constexpr static auto kFixedArrayUseDefault = static_cast<size_t>(-1); // A `FixedArray` provides a run-time fixed-size array, allocating a small array // inline for efficiency. // -// Most users should not specify an `inline_elements` argument and let -// `FixedArray` automatically determine the number of elements -// to store inline based on `sizeof(T)`. If `inline_elements` is specified, the -// `FixedArray` implementation will use inline storage for arrays with a -// length <= `inline_elements`. +// Most users should not specify the `N` template parameter and let `FixedArray` +// automatically determine the number of elements to store inline based on +// `sizeof(T)`. If `N` is specified, the `FixedArray` implementation will use +// inline storage for arrays with a length <= `N`. // // Note that a `FixedArray` constructed with a `size_type` argument will // default-initialize its values by leaving trivially constructible types @@ -471,6 +470,9 @@ class FixedArray { return n <= inline_elements; } +#ifdef Y_ABSL_HAVE_ADDRESS_SANITIZER + Y_ABSL_ATTRIBUTE_NOINLINE +#endif // Y_ABSL_HAVE_ADDRESS_SANITIZER StorageElement* InitializeData() { if (UsingInlinedStorage(size())) { InlinedStorage::AnnotateConstruct(size()); diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/container/inlined_vector.h b/contrib/restricted/abseil-cpp-tstring/y_absl/container/inlined_vector.h index fd5d07019a..aaec71987e 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/container/inlined_vector.h +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/container/inlined_vector.h @@ -52,6 +52,7 @@ #include "y_absl/base/port.h" #include "y_absl/container/internal/inlined_vector.h" #include "y_absl/memory/memory.h" +#include "y_absl/meta/type_traits.h" namespace y_absl { Y_ABSL_NAMESPACE_BEGIN @@ -77,6 +78,8 @@ class InlinedVector { using MoveIterator = inlined_vector_internal::MoveIterator<TheA>; template <typename TheA> using IsMemcpyOk = inlined_vector_internal::IsMemcpyOk<TheA>; + template <typename TheA> + using IsMoveAssignOk = inlined_vector_internal::IsMoveAssignOk<TheA>; template <typename TheA, typename Iterator> using IteratorValueAdapter = @@ -94,6 +97,12 @@ class InlinedVector { using DisableIfAtLeastForwardIterator = y_absl::enable_if_t< !inlined_vector_internal::IsAtLeastForwardIterator<Iterator>::value, int>; + using MemcpyPolicy = typename Storage::MemcpyPolicy; + using ElementwiseAssignPolicy = typename Storage::ElementwiseAssignPolicy; + using ElementwiseConstructPolicy = + typename Storage::ElementwiseConstructPolicy; + using MoveAssignmentPolicy = typename Storage::MoveAssignmentPolicy; + public: using allocator_type = A; using value_type = inlined_vector_internal::ValueType<A>; @@ -275,8 +284,10 @@ class InlinedVector { size_type max_size() const noexcept { // One bit of the size storage is used to indicate whether the inlined // vector contains allocated memory. As a result, the maximum size that the - // inlined vector can express is half of the max for `size_type`. - return (std::numeric_limits<size_type>::max)() / 2; + // inlined vector can express is the minimum of the limit of how many + // objects we can allocate and std::numeric_limits<size_type>::max() / 2. + return (std::min)(AllocatorTraits<A>::max_size(storage_.GetAllocator()), + (std::numeric_limits<size_type>::max)() / 2); } // `InlinedVector::capacity()` @@ -484,18 +495,7 @@ class InlinedVector { // unspecified state. InlinedVector& operator=(InlinedVector&& other) { if (Y_ABSL_PREDICT_TRUE(this != std::addressof(other))) { - if (IsMemcpyOk<A>::value || other.storage_.GetIsAllocated()) { - inlined_vector_internal::DestroyAdapter<A>::DestroyElements( - storage_.GetAllocator(), data(), size()); - storage_.DeallocateIfAllocated(); - storage_.MemcpyFrom(other.storage_); - - other.storage_.SetInlinedSize(0); - } else { - storage_.Assign(IteratorValueAdapter<A, MoveIterator<A>>( - MoveIterator<A>(other.storage_.GetInlinedData())), - other.size()); - } + MoveAssignment(MoveAssignmentPolicy{}, std::move(other)); } return *this; @@ -624,9 +624,9 @@ class InlinedVector { Y_ABSL_HARDENING_ASSERT(pos <= end()); if (Y_ABSL_PREDICT_TRUE(first != last)) { - return storage_.Insert(pos, - IteratorValueAdapter<A, ForwardIterator>(first), - std::distance(first, last)); + return storage_.Insert( + pos, IteratorValueAdapter<A, ForwardIterator>(first), + static_cast<size_type>(std::distance(first, last))); } else { return const_cast<iterator>(pos); } @@ -643,7 +643,7 @@ class InlinedVector { Y_ABSL_HARDENING_ASSERT(pos >= begin()); Y_ABSL_HARDENING_ASSERT(pos <= end()); - size_type index = std::distance(cbegin(), pos); + size_type index = static_cast<size_type>(std::distance(cbegin(), pos)); for (size_type i = index; first != last; ++i, static_cast<void>(++first)) { insert(data() + i, *first); } @@ -661,10 +661,22 @@ class InlinedVector { Y_ABSL_HARDENING_ASSERT(pos <= end()); value_type dealias(std::forward<Args>(args)...); + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102329#c2 + // It appears that GCC thinks that since `pos` is a const pointer and may + // point to uninitialized memory at this point, a warning should be + // issued. But `pos` is actually only used to compute an array index to + // write to. +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif return storage_.Insert(pos, IteratorValueAdapter<A, MoveIterator<A>>( MoveIterator<A>(std::addressof(dealias))), 1); +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic pop +#endif } // `InlinedVector::emplace_back(...)` @@ -771,6 +783,42 @@ class InlinedVector { template <typename H, typename TheT, size_t TheN, typename TheA> friend H AbslHashValue(H h, const y_absl::InlinedVector<TheT, TheN, TheA>& a); + void MoveAssignment(MemcpyPolicy, InlinedVector&& other) { + inlined_vector_internal::DestroyAdapter<A>::DestroyElements( + storage_.GetAllocator(), data(), size()); + storage_.DeallocateIfAllocated(); + storage_.MemcpyFrom(other.storage_); + + other.storage_.SetInlinedSize(0); + } + + void MoveAssignment(ElementwiseAssignPolicy, InlinedVector&& other) { + if (other.storage_.GetIsAllocated()) { + MoveAssignment(MemcpyPolicy{}, std::move(other)); + } else { + storage_.Assign(IteratorValueAdapter<A, MoveIterator<A>>( + MoveIterator<A>(other.storage_.GetInlinedData())), + other.size()); + } + } + + void MoveAssignment(ElementwiseConstructPolicy, InlinedVector&& other) { + if (other.storage_.GetIsAllocated()) { + MoveAssignment(MemcpyPolicy{}, std::move(other)); + } else { + inlined_vector_internal::DestroyAdapter<A>::DestroyElements( + storage_.GetAllocator(), data(), size()); + storage_.DeallocateIfAllocated(); + + IteratorValueAdapter<A, MoveIterator<A>> other_values( + MoveIterator<A>(other.storage_.GetInlinedData())); + inlined_vector_internal::ConstructElements<A>( + storage_.GetAllocator(), storage_.GetInlinedData(), other_values, + other.storage_.GetSize()); + storage_.SetInlinedSize(other.storage_.GetSize()); + } + } + Storage storage_; }; diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/common.h b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/common.h index c32dbd8179..4fc2675f4f 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/common.h +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/common.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef Y_ABSL_CONTAINER_INTERNAL_CONTAINER_H_ -#define Y_ABSL_CONTAINER_INTERNAL_CONTAINER_H_ +#ifndef Y_ABSL_CONTAINER_INTERNAL_COMMON_H_ +#define Y_ABSL_CONTAINER_INTERNAL_COMMON_H_ #include <cassert> #include <type_traits> @@ -204,4 +204,4 @@ struct InsertReturnType { Y_ABSL_NAMESPACE_END } // namespace y_absl -#endif // Y_ABSL_CONTAINER_INTERNAL_CONTAINER_H_ +#endif // Y_ABSL_CONTAINER_INTERNAL_COMMON_H_ diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/common_policy_traits.h b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/common_policy_traits.h new file mode 100644 index 0000000000..927f6dac2d --- /dev/null +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/common_policy_traits.h @@ -0,0 +1,132 @@ +// Copyright 2022 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef Y_ABSL_CONTAINER_INTERNAL_COMMON_POLICY_TRAITS_H_ +#define Y_ABSL_CONTAINER_INTERNAL_COMMON_POLICY_TRAITS_H_ + +#include <cstddef> +#include <cstring> +#include <memory> +#include <new> +#include <type_traits> +#include <utility> + +#include "y_absl/meta/type_traits.h" + +namespace y_absl { +Y_ABSL_NAMESPACE_BEGIN +namespace container_internal { + +// Defines how slots are initialized/destroyed/moved. +template <class Policy, class = void> +struct common_policy_traits { + // The actual object stored in the container. + using slot_type = typename Policy::slot_type; + using reference = decltype(Policy::element(std::declval<slot_type*>())); + using value_type = typename std::remove_reference<reference>::type; + + // PRECONDITION: `slot` is UNINITIALIZED + // POSTCONDITION: `slot` is INITIALIZED + template <class Alloc, class... Args> + static void construct(Alloc* alloc, slot_type* slot, Args&&... args) { + Policy::construct(alloc, slot, std::forward<Args>(args)...); + } + + // PRECONDITION: `slot` is INITIALIZED + // POSTCONDITION: `slot` is UNINITIALIZED + template <class Alloc> + static void destroy(Alloc* alloc, slot_type* slot) { + Policy::destroy(alloc, slot); + } + + // Transfers the `old_slot` to `new_slot`. Any memory allocated by the + // allocator inside `old_slot` to `new_slot` can be transferred. + // + // OPTIONAL: defaults to: + // + // clone(new_slot, std::move(*old_slot)); + // destroy(old_slot); + // + // PRECONDITION: `new_slot` is UNINITIALIZED and `old_slot` is INITIALIZED + // POSTCONDITION: `new_slot` is INITIALIZED and `old_slot` is + // UNINITIALIZED + template <class Alloc> + static void transfer(Alloc* alloc, slot_type* new_slot, slot_type* old_slot) { + transfer_impl(alloc, new_slot, old_slot, Rank0{}); + } + + // PRECONDITION: `slot` is INITIALIZED + // POSTCONDITION: `slot` is INITIALIZED + // Note: we use remove_const_t so that the two overloads have different args + // in the case of sets with explicitly const value_types. + template <class P = Policy> + static auto element(y_absl::remove_const_t<slot_type>* slot) + -> decltype(P::element(slot)) { + return P::element(slot); + } + template <class P = Policy> + static auto element(const slot_type* slot) -> decltype(P::element(slot)) { + return P::element(slot); + } + + static constexpr bool transfer_uses_memcpy() { + return std::is_same<decltype(transfer_impl<std::allocator<char>>( + nullptr, nullptr, nullptr, Rank0{})), + std::true_type>::value; + } + + private: + // To rank the overloads below for overload resoltion. Rank0 is preferred. + struct Rank2 {}; + struct Rank1 : Rank2 {}; + struct Rank0 : Rank1 {}; + + // Use auto -> decltype as an enabler. + template <class Alloc, class P = Policy> + static auto transfer_impl(Alloc* alloc, slot_type* new_slot, + slot_type* old_slot, Rank0) + -> decltype((void)P::transfer(alloc, new_slot, old_slot)) { + P::transfer(alloc, new_slot, old_slot); + } +#if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606 + // This overload returns true_type for the trait below. + // The conditional_t is to make the enabler type dependent. + template <class Alloc, + typename = std::enable_if_t<y_absl::is_trivially_relocatable< + std::conditional_t<false, Alloc, value_type>>::value>> + static std::true_type transfer_impl(Alloc*, slot_type* new_slot, + slot_type* old_slot, Rank1) { + // TODO(b/247130232): remove casts after fixing warnings. + // TODO(b/251814870): remove casts after fixing warnings. + std::memcpy( + static_cast<void*>(std::launder( + const_cast<std::remove_const_t<value_type>*>(&element(new_slot)))), + static_cast<const void*>(&element(old_slot)), sizeof(value_type)); + return {}; + } +#endif + + template <class Alloc> + static void transfer_impl(Alloc* alloc, slot_type* new_slot, + slot_type* old_slot, Rank2) { + construct(alloc, new_slot, std::move(element(old_slot))); + destroy(alloc, old_slot); + } +}; + +} // namespace container_internal +Y_ABSL_NAMESPACE_END +} // namespace y_absl + +#endif // Y_ABSL_CONTAINER_INTERNAL_COMMON_POLICY_TRAITS_H_ diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/container_memory.h b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/container_memory.h index d40cf7531c..9a44a3933e 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/container_memory.h +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/container_memory.h @@ -17,6 +17,7 @@ #include <cassert> #include <cstddef> +#include <cstring> #include <memory> #include <new> #include <tuple> @@ -340,7 +341,8 @@ template <class K, class V> struct map_slot_policy { using slot_type = map_slot_type<K, V>; using value_type = std::pair<const K, V>; - using mutable_value_type = std::pair<K, V>; + using mutable_value_type = + std::pair<y_absl::remove_const_t<K>, y_absl::remove_const_t<V>>; private: static void emplace(slot_type* slot) { @@ -424,6 +426,16 @@ struct map_slot_policy { static void transfer(Allocator* alloc, slot_type* new_slot, slot_type* old_slot) { emplace(new_slot); +#if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606 + if (y_absl::is_trivially_relocatable<value_type>()) { + // TODO(b/247130232,b/251814870): remove casts after fixing warnings. + std::memcpy(static_cast<void*>(std::launder(&new_slot->value)), + static_cast<const void*>(&old_slot->value), + sizeof(value_type)); + return; + } +#endif + if (kMutableKeys::value) { y_absl::allocator_traits<Allocator>::construct( *alloc, &new_slot->mutable_value, std::move(old_slot->mutable_value)); diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/hash_policy_traits.h b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/hash_policy_traits.h index 1ac62b0a3e..cf60b63e5f 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/hash_policy_traits.h +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/hash_policy_traits.h @@ -21,6 +21,7 @@ #include <type_traits> #include <utility> +#include "y_absl/container/internal/common_policy_traits.h" #include "y_absl/meta/type_traits.h" namespace y_absl { @@ -29,7 +30,7 @@ namespace container_internal { // Defines how slots are initialized/destroyed/moved. template <class Policy, class = void> -struct hash_policy_traits { +struct hash_policy_traits : common_policy_traits<Policy> { // The type of the keys stored in the hashtable. using key_type = typename Policy::key_type; @@ -87,43 +88,6 @@ struct hash_policy_traits { // Defaults to false if not provided by the policy. using constant_iterators = ConstantIteratorsImpl<>; - // PRECONDITION: `slot` is UNINITIALIZED - // POSTCONDITION: `slot` is INITIALIZED - template <class Alloc, class... Args> - static void construct(Alloc* alloc, slot_type* slot, Args&&... args) { - Policy::construct(alloc, slot, std::forward<Args>(args)...); - } - - // PRECONDITION: `slot` is INITIALIZED - // POSTCONDITION: `slot` is UNINITIALIZED - template <class Alloc> - static void destroy(Alloc* alloc, slot_type* slot) { - Policy::destroy(alloc, slot); - } - - // Transfers the `old_slot` to `new_slot`. Any memory allocated by the - // allocator inside `old_slot` to `new_slot` can be transferred. - // - // OPTIONAL: defaults to: - // - // clone(new_slot, std::move(*old_slot)); - // destroy(old_slot); - // - // PRECONDITION: `new_slot` is UNINITIALIZED and `old_slot` is INITIALIZED - // POSTCONDITION: `new_slot` is INITIALIZED and `old_slot` is - // UNINITIALIZED - template <class Alloc> - static void transfer(Alloc* alloc, slot_type* new_slot, slot_type* old_slot) { - transfer_impl(alloc, new_slot, old_slot, 0); - } - - // PRECONDITION: `slot` is INITIALIZED - // POSTCONDITION: `slot` is INITIALIZED - template <class P = Policy> - static auto element(slot_type* slot) -> decltype(P::element(slot)) { - return P::element(slot); - } - // Returns the amount of memory owned by `slot`, exclusive of `sizeof(*slot)`. // // If `slot` is nullptr, returns the constant amount of memory owned by any @@ -174,8 +138,8 @@ struct hash_policy_traits { // Used for node handle manipulation. template <class P = Policy> static auto mutable_key(slot_type* slot) - -> decltype(P::apply(ReturnKey(), element(slot))) { - return P::apply(ReturnKey(), element(slot)); + -> decltype(P::apply(ReturnKey(), hash_policy_traits::element(slot))) { + return P::apply(ReturnKey(), hash_policy_traits::element(slot)); } // Returns the "value" (as opposed to the "key") portion of the element. Used @@ -184,21 +148,6 @@ struct hash_policy_traits { static auto value(T* elem) -> decltype(P::value(elem)) { return P::value(elem); } - - private: - // Use auto -> decltype as an enabler. - template <class Alloc, class P = Policy> - static auto transfer_impl(Alloc* alloc, slot_type* new_slot, - slot_type* old_slot, int) - -> decltype((void)P::transfer(alloc, new_slot, old_slot)) { - P::transfer(alloc, new_slot, old_slot); - } - template <class Alloc> - static void transfer_impl(Alloc* alloc, slot_type* new_slot, - slot_type* old_slot, char) { - construct(alloc, new_slot, std::move(element(old_slot))); - destroy(alloc, old_slot); - } }; } // namespace container_internal diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/hashtablez_sampler.cc b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/hashtablez_sampler.cc index 36816872fd..d58a22a0b2 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/hashtablez_sampler.cc +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/hashtablez_sampler.cc @@ -14,6 +14,7 @@ #include "y_absl/container/internal/hashtablez_sampler.h" +#include <algorithm> #include <atomic> #include <cassert> #include <cmath> @@ -158,6 +159,43 @@ void UnsampleSlow(HashtablezInfo* info) { GlobalHashtablezSampler().Unregister(info); } +void RecordRehashSlow(HashtablezInfo* info, size_t total_probe_length) { +#ifdef Y_ABSL_INTERNAL_HAVE_SSE2 + total_probe_length /= 16; +#else + total_probe_length /= 8; +#endif + info->total_probe_length.store(total_probe_length, std::memory_order_relaxed); + info->num_erases.store(0, std::memory_order_relaxed); + // There is only one concurrent writer, so `load` then `store` is sufficient + // instead of using `fetch_add`. + info->num_rehashes.store( + 1 + info->num_rehashes.load(std::memory_order_relaxed), + std::memory_order_relaxed); +} + +void RecordReservationSlow(HashtablezInfo* info, size_t target_capacity) { + info->max_reserve.store( + (std::max)(info->max_reserve.load(std::memory_order_relaxed), + target_capacity), + std::memory_order_relaxed); +} + +void RecordClearedReservationSlow(HashtablezInfo* info) { + info->max_reserve.store(0, std::memory_order_relaxed); +} + +void RecordStorageChangedSlow(HashtablezInfo* info, size_t size, + size_t capacity) { + info->size.store(size, std::memory_order_relaxed); + info->capacity.store(capacity, std::memory_order_relaxed); + if (size == 0) { + // This is a clear, reset the total/num_erases too. + info->total_probe_length.store(0, std::memory_order_relaxed); + info->num_erases.store(0, std::memory_order_relaxed); + } +} + void RecordInsertSlow(HashtablezInfo* info, size_t hash, size_t distance_from_desired) { // SwissTables probe in groups of 16, so scale this to count items probes and @@ -180,6 +218,14 @@ void RecordInsertSlow(HashtablezInfo* info, size_t hash, info->size.fetch_add(1, std::memory_order_relaxed); } +void RecordEraseSlow(HashtablezInfo* info) { + info->size.fetch_sub(1, std::memory_order_relaxed); + // There is only one concurrent writer, so `load` then `store` is sufficient + // instead of using `fetch_add`. + info->num_erases.store(1 + info->num_erases.load(std::memory_order_relaxed), + std::memory_order_relaxed); +} + void SetHashtablezConfigListener(HashtablezConfigListener l) { g_hashtablez_config_listener.store(l, std::memory_order_release); } @@ -215,21 +261,20 @@ void SetHashtablezSampleParameterInternal(int32_t rate) { } } -int32_t GetHashtablezMaxSamples() { +size_t GetHashtablezMaxSamples() { return GlobalHashtablezSampler().GetMaxSamples(); } -void SetHashtablezMaxSamples(int32_t max) { +void SetHashtablezMaxSamples(size_t max) { SetHashtablezMaxSamplesInternal(max); TriggerHashtablezConfigListener(); } -void SetHashtablezMaxSamplesInternal(int32_t max) { +void SetHashtablezMaxSamplesInternal(size_t max) { if (max > 0) { GlobalHashtablezSampler().SetMaxSamples(max); } else { - Y_ABSL_RAW_LOG(ERROR, "Invalid hashtablez max samples: %lld", - static_cast<long long>(max)); // NOLINT(runtime/int) + Y_ABSL_RAW_LOG(ERROR, "Invalid hashtablez max samples: 0"); } } diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/hashtablez_sampler.h b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/hashtablez_sampler.h index fefa2bb9f0..95353663f2 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/hashtablez_sampler.h +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/hashtablez_sampler.h @@ -95,55 +95,19 @@ struct HashtablezInfo : public profiling_internal::Sample<HashtablezInfo> { size_t inline_element_size; // How big is the slot? }; -inline void RecordRehashSlow(HashtablezInfo* info, size_t total_probe_length) { -#ifdef Y_ABSL_INTERNAL_HAVE_SSE2 - total_probe_length /= 16; -#else - total_probe_length /= 8; -#endif - info->total_probe_length.store(total_probe_length, std::memory_order_relaxed); - info->num_erases.store(0, std::memory_order_relaxed); - // There is only one concurrent writer, so `load` then `store` is sufficient - // instead of using `fetch_add`. - info->num_rehashes.store( - 1 + info->num_rehashes.load(std::memory_order_relaxed), - std::memory_order_relaxed); -} +void RecordRehashSlow(HashtablezInfo* info, size_t total_probe_length); -inline void RecordReservationSlow(HashtablezInfo* info, - size_t target_capacity) { - info->max_reserve.store( - (std::max)(info->max_reserve.load(std::memory_order_relaxed), - target_capacity), - std::memory_order_relaxed); -} +void RecordReservationSlow(HashtablezInfo* info, size_t target_capacity); -inline void RecordClearedReservationSlow(HashtablezInfo* info) { - info->max_reserve.store(0, std::memory_order_relaxed); -} +void RecordClearedReservationSlow(HashtablezInfo* info); -inline void RecordStorageChangedSlow(HashtablezInfo* info, size_t size, - size_t capacity) { - info->size.store(size, std::memory_order_relaxed); - info->capacity.store(capacity, std::memory_order_relaxed); - if (size == 0) { - // This is a clear, reset the total/num_erases too. - info->total_probe_length.store(0, std::memory_order_relaxed); - info->num_erases.store(0, std::memory_order_relaxed); - } -} +void RecordStorageChangedSlow(HashtablezInfo* info, size_t size, + size_t capacity); void RecordInsertSlow(HashtablezInfo* info, size_t hash, size_t distance_from_desired); -inline void RecordEraseSlow(HashtablezInfo* info) { - info->size.fetch_sub(1, std::memory_order_relaxed); - // There is only one concurrent writer, so `load` then `store` is sufficient - // instead of using `fetch_add`. - info->num_erases.store( - 1 + info->num_erases.load(std::memory_order_relaxed), - std::memory_order_relaxed); -} +void RecordEraseSlow(HashtablezInfo* info); struct SamplingState { int64_t next_sample; @@ -165,7 +129,10 @@ class HashtablezInfoHandle { public: explicit HashtablezInfoHandle() : info_(nullptr) {} explicit HashtablezInfoHandle(HashtablezInfo* info) : info_(info) {} - ~HashtablezInfoHandle() { + + // We do not have a destructor. Caller is responsible for calling Unregister + // before destroying the handle. + void Unregister() { if (Y_ABSL_PREDICT_TRUE(info_ == nullptr)) return; UnsampleSlow(info_); } @@ -230,6 +197,7 @@ class HashtablezInfoHandle { explicit HashtablezInfoHandle() = default; explicit HashtablezInfoHandle(std::nullptr_t) {} + inline void Unregister() {} inline void RecordStorageChanged(size_t /*size*/, size_t /*capacity*/) {} inline void RecordRehash(size_t /*total_probe_length*/) {} inline void RecordReservation(size_t /*target_capacity*/) {} @@ -281,9 +249,9 @@ void SetHashtablezSampleParameter(int32_t rate); void SetHashtablezSampleParameterInternal(int32_t rate); // Sets a soft max for the number of samples that will be kept. -int32_t GetHashtablezMaxSamples(); -void SetHashtablezMaxSamples(int32_t max); -void SetHashtablezMaxSamplesInternal(int32_t max); +size_t GetHashtablezMaxSamples(); +void SetHashtablezMaxSamples(size_t max); +void SetHashtablezMaxSamplesInternal(size_t max); // Configuration override. // This allows process-wide sampling without depending on order of diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/inlined_vector.h b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/inlined_vector.h index 047dd99882..b90c16528d 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/inlined_vector.h +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/inlined_vector.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef Y_ABSL_CONTAINER_INTERNAL_INLINED_VECTOR_INTERNAL_H_ -#define Y_ABSL_CONTAINER_INTERNAL_INLINED_VECTOR_INTERNAL_H_ +#ifndef Y_ABSL_CONTAINER_INTERNAL_INLINED_VECTOR_H_ +#define Y_ABSL_CONTAINER_INTERNAL_INLINED_VECTOR_H_ #include <algorithm> #include <cstddef> @@ -83,6 +83,11 @@ using IsMemcpyOk = y_absl::is_trivially_copy_assignable<ValueType<A>>, y_absl::is_trivially_destructible<ValueType<A>>>; +template <typename A> +using IsMoveAssignOk = std::is_move_assignable<ValueType<A>>; +template <typename A> +using IsSwapOk = y_absl::type_traits_internal::IsSwappable<ValueType<A>>; + template <typename T> struct TypeIdentity { using type = T; @@ -120,8 +125,8 @@ struct DestroyAdapter<A, /* IsTriviallyDestructible */ true> { template <typename A> struct Allocation { - Pointer<A> data; - SizeType<A> capacity; + Pointer<A> data = nullptr; + SizeType<A> capacity = 0; }; template <typename A, @@ -297,6 +302,20 @@ class ConstructionTransaction { template <typename T, size_t N, typename A> class Storage { public: + struct MemcpyPolicy {}; + struct ElementwiseAssignPolicy {}; + struct ElementwiseSwapPolicy {}; + struct ElementwiseConstructPolicy {}; + + using MoveAssignmentPolicy = y_absl::conditional_t< + IsMemcpyOk<A>::value, MemcpyPolicy, + y_absl::conditional_t<IsMoveAssignOk<A>::value, ElementwiseAssignPolicy, + ElementwiseConstructPolicy>>; + using SwapPolicy = y_absl::conditional_t< + IsMemcpyOk<A>::value, MemcpyPolicy, + y_absl::conditional_t<IsSwapOk<A>::value, ElementwiseSwapPolicy, + ElementwiseConstructPolicy>>; + static SizeType<A> NextCapacity(SizeType<A> current_capacity) { return current_capacity * 2; } @@ -360,7 +379,9 @@ class Storage { return data_.allocated.allocated_capacity; } - SizeType<A> GetInlinedCapacity() const { return static_cast<SizeType<A>>(N); } + SizeType<A> GetInlinedCapacity() const { + return static_cast<SizeType<A>>(kOptimalInlinedSize); + } StorageView<A> MakeStorageView() { return GetIsAllocated() ? StorageView<A>{GetAllocatedData(), GetSize(), @@ -464,8 +485,15 @@ class Storage { SizeType<A> allocated_capacity; }; + // `kOptimalInlinedSize` is an automatically adjusted inlined capacity of the + // `InlinedVector`. Sometimes, it is possible to increase the capacity (from + // the user requested `N`) without increasing the size of the `InlinedVector`. + static constexpr size_t kOptimalInlinedSize = + (std::max)(N, sizeof(Allocated) / sizeof(ValueType<A>)); + struct Inlined { - alignas(ValueType<A>) char inlined_data[sizeof(ValueType<A>[N])]; + alignas(ValueType<A>) char inlined_data[sizeof( + ValueType<A>[kOptimalInlinedSize])]; }; union Data { @@ -473,6 +501,13 @@ class Storage { Inlined inlined; }; + void SwapN(ElementwiseSwapPolicy, Storage* other, SizeType<A> n); + void SwapN(ElementwiseConstructPolicy, Storage* other, SizeType<A> n); + + void SwapInlinedElements(MemcpyPolicy, Storage* other); + template <typename NotMemcpyPolicy> + void SwapInlinedElements(NotMemcpyPolicy, Storage* other); + template <typename... Args> Y_ABSL_ATTRIBUTE_NOINLINE Reference<A> EmplaceBackSlow(Args&&... args); @@ -641,8 +676,8 @@ auto Storage<T, N, A>::Insert(ConstIterator<A> pos, ValueAdapter values, SizeType<A> insert_count) -> Iterator<A> { StorageView<A> storage_view = MakeStorageView(); - SizeType<A> insert_index = - std::distance(ConstIterator<A>(storage_view.data), pos); + auto insert_index = static_cast<SizeType<A>>( + std::distance(ConstIterator<A>(storage_view.data), pos)); SizeType<A> insert_end_index = insert_index + insert_count; SizeType<A> new_size = storage_view.size + insert_count; @@ -784,9 +819,9 @@ auto Storage<T, N, A>::Erase(ConstIterator<A> from, ConstIterator<A> to) -> Iterator<A> { StorageView<A> storage_view = MakeStorageView(); - SizeType<A> erase_size = std::distance(from, to); - SizeType<A> erase_index = - std::distance(ConstIterator<A>(storage_view.data), from); + auto erase_size = static_cast<SizeType<A>>(std::distance(from, to)); + auto erase_index = static_cast<SizeType<A>>( + std::distance(ConstIterator<A>(storage_view.data), from)); SizeType<A> erase_end_index = erase_index + erase_size; IteratorValueAdapter<A, MoveIterator<A>> move_values( @@ -886,26 +921,7 @@ auto Storage<T, N, A>::Swap(Storage* other_storage_ptr) -> void { if (GetIsAllocated() && other_storage_ptr->GetIsAllocated()) { swap(data_.allocated, other_storage_ptr->data_.allocated); } else if (!GetIsAllocated() && !other_storage_ptr->GetIsAllocated()) { - Storage* small_ptr = this; - Storage* large_ptr = other_storage_ptr; - if (small_ptr->GetSize() > large_ptr->GetSize()) swap(small_ptr, large_ptr); - - for (SizeType<A> i = 0; i < small_ptr->GetSize(); ++i) { - swap(small_ptr->GetInlinedData()[i], large_ptr->GetInlinedData()[i]); - } - - IteratorValueAdapter<A, MoveIterator<A>> move_values( - MoveIterator<A>(large_ptr->GetInlinedData() + small_ptr->GetSize())); - - ConstructElements<A>(large_ptr->GetAllocator(), - small_ptr->GetInlinedData() + small_ptr->GetSize(), - move_values, - large_ptr->GetSize() - small_ptr->GetSize()); - - DestroyAdapter<A>::DestroyElements( - large_ptr->GetAllocator(), - large_ptr->GetInlinedData() + small_ptr->GetSize(), - large_ptr->GetSize() - small_ptr->GetSize()); + SwapInlinedElements(SwapPolicy{}, other_storage_ptr); } else { Storage* allocated_ptr = this; Storage* inlined_ptr = other_storage_ptr; @@ -941,6 +957,68 @@ auto Storage<T, N, A>::Swap(Storage* other_storage_ptr) -> void { swap(GetAllocator(), other_storage_ptr->GetAllocator()); } +template <typename T, size_t N, typename A> +void Storage<T, N, A>::SwapN(ElementwiseSwapPolicy, Storage* other, + SizeType<A> n) { + std::swap_ranges(GetInlinedData(), GetInlinedData() + n, + other->GetInlinedData()); +} + +template <typename T, size_t N, typename A> +void Storage<T, N, A>::SwapN(ElementwiseConstructPolicy, Storage* other, + SizeType<A> n) { + Pointer<A> a = GetInlinedData(); + Pointer<A> b = other->GetInlinedData(); + // see note on allocators in `SwapInlinedElements`. + A& allocator_a = GetAllocator(); + A& allocator_b = other->GetAllocator(); + for (SizeType<A> i = 0; i < n; ++i, ++a, ++b) { + ValueType<A> tmp(std::move(*a)); + + AllocatorTraits<A>::destroy(allocator_a, a); + AllocatorTraits<A>::construct(allocator_b, a, std::move(*b)); + + AllocatorTraits<A>::destroy(allocator_b, b); + AllocatorTraits<A>::construct(allocator_a, b, std::move(tmp)); + } +} + +template <typename T, size_t N, typename A> +void Storage<T, N, A>::SwapInlinedElements(MemcpyPolicy, Storage* other) { + Data tmp = data_; + data_ = other->data_; + other->data_ = tmp; +} + +template <typename T, size_t N, typename A> +template <typename NotMemcpyPolicy> +void Storage<T, N, A>::SwapInlinedElements(NotMemcpyPolicy policy, + Storage* other) { + // Note: `destroy` needs to use pre-swap allocator while `construct` - + // post-swap allocator. Allocators will be swaped later on outside of + // `SwapInlinedElements`. + Storage* small_ptr = this; + Storage* large_ptr = other; + if (small_ptr->GetSize() > large_ptr->GetSize()) { + std::swap(small_ptr, large_ptr); + } + + auto small_size = small_ptr->GetSize(); + auto diff = large_ptr->GetSize() - small_size; + SwapN(policy, other, small_size); + + IteratorValueAdapter<A, MoveIterator<A>> move_values( + MoveIterator<A>(large_ptr->GetInlinedData() + small_size)); + + ConstructElements<A>(large_ptr->GetAllocator(), + small_ptr->GetInlinedData() + small_size, move_values, + diff); + + DestroyAdapter<A>::DestroyElements(large_ptr->GetAllocator(), + large_ptr->GetInlinedData() + small_size, + diff); +} + // End ignore "array-bounds" #if !defined(__clang__) && defined(__GNUC__) #pragma GCC diagnostic pop @@ -950,4 +1028,4 @@ auto Storage<T, N, A>::Swap(Storage* other_storage_ptr) -> void { Y_ABSL_NAMESPACE_END } // namespace y_absl -#endif // Y_ABSL_CONTAINER_INTERNAL_INLINED_VECTOR_INTERNAL_H_ +#endif // Y_ABSL_CONTAINER_INTERNAL_INLINED_VECTOR_H_ diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/raw_hash_set.cc b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/raw_hash_set.cc index d18535fcbb..e41730f431 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/raw_hash_set.cc +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/raw_hash_set.cc @@ -16,6 +16,7 @@ #include <atomic> #include <cstddef> +#include <cstring> #include "y_absl/base/config.h" @@ -25,11 +26,14 @@ namespace container_internal { // A single block of empty control bytes for tables without any slots allocated. // This enables removing a branch in the hot path of find(). -alignas(16) Y_ABSL_CONST_INIT Y_ABSL_DLL const ctrl_t kEmptyGroup[16] = { +// We have 17 bytes because there may be a generation counter. Any constant is +// fine for the generation counter. +alignas(16) Y_ABSL_CONST_INIT Y_ABSL_DLL const ctrl_t kEmptyGroup[17] = { ctrl_t::kSentinel, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, - ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty}; + ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, + static_cast<ctrl_t>(0)}; #ifdef Y_ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL constexpr size_t Group::kWidth; @@ -63,8 +67,155 @@ void ConvertDeletedToEmptyAndFullToDeleted(ctrl_t* ctrl, size_t capacity) { std::memcpy(ctrl + capacity + 1, ctrl, NumClonedBytes()); ctrl[capacity] = ctrl_t::kSentinel; } -// Extern template instantiotion for inline function. -template FindInfo find_first_non_full(const ctrl_t*, size_t, size_t); +// Extern template instantiation for inline function. +template FindInfo find_first_non_full(const CommonFields&, size_t); + +FindInfo find_first_non_full_outofline(const CommonFields& common, + size_t hash) { + return find_first_non_full(common, hash); +} + +// Return address of the ith slot in slots where each slot occupies slot_size. +static inline void* SlotAddress(void* slot_array, size_t slot, + size_t slot_size) { + return reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(slot_array) + + (slot * slot_size)); +} + +// Return the address of the slot just after slot assuming each slot +// has the specified size. +static inline void* NextSlot(void* slot, size_t slot_size) { + return reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(slot) + slot_size); +} + +// Return the address of the slot just before slot assuming each slot +// has the specified size. +static inline void* PrevSlot(void* slot, size_t slot_size) { + return reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(slot) - slot_size); +} + +void DropDeletesWithoutResize(CommonFields& common, + const PolicyFunctions& policy, void* tmp_space) { + void* set = &common; + void* slot_array = common.slots_; + const size_t capacity = common.capacity_; + assert(IsValidCapacity(capacity)); + assert(!is_small(capacity)); + // Algorithm: + // - mark all DELETED slots as EMPTY + // - mark all FULL slots as DELETED + // - for each slot marked as DELETED + // hash = Hash(element) + // target = find_first_non_full(hash) + // if target is in the same group + // mark slot as FULL + // else if target is EMPTY + // transfer element to target + // mark slot as EMPTY + // mark target as FULL + // else if target is DELETED + // swap current element with target element + // mark target as FULL + // repeat procedure for current slot with moved from element (target) + ctrl_t* ctrl = common.control_; + ConvertDeletedToEmptyAndFullToDeleted(ctrl, capacity); + auto hasher = policy.hash_slot; + auto transfer = policy.transfer; + const size_t slot_size = policy.slot_size; + + size_t total_probe_length = 0; + void* slot_ptr = SlotAddress(slot_array, 0, slot_size); + for (size_t i = 0; i != capacity; + ++i, slot_ptr = NextSlot(slot_ptr, slot_size)) { + assert(slot_ptr == SlotAddress(slot_array, i, slot_size)); + if (!IsDeleted(ctrl[i])) continue; + const size_t hash = (*hasher)(set, slot_ptr); + const FindInfo target = find_first_non_full(common, hash); + const size_t new_i = target.offset; + total_probe_length += target.probe_length; + + // Verify if the old and new i fall within the same group wrt the hash. + // If they do, we don't need to move the object as it falls already in the + // best probe we can. + const size_t probe_offset = probe(common, hash).offset(); + const auto probe_index = [probe_offset, capacity](size_t pos) { + return ((pos - probe_offset) & capacity) / Group::kWidth; + }; + + // Element doesn't move. + if (Y_ABSL_PREDICT_TRUE(probe_index(new_i) == probe_index(i))) { + SetCtrl(common, i, H2(hash), slot_size); + continue; + } + + void* new_slot_ptr = SlotAddress(slot_array, new_i, slot_size); + if (IsEmpty(ctrl[new_i])) { + // Transfer element to the empty spot. + // SetCtrl poisons/unpoisons the slots so we have to call it at the + // right time. + SetCtrl(common, new_i, H2(hash), slot_size); + (*transfer)(set, new_slot_ptr, slot_ptr); + SetCtrl(common, i, ctrl_t::kEmpty, slot_size); + } else { + assert(IsDeleted(ctrl[new_i])); + SetCtrl(common, new_i, H2(hash), slot_size); + // Until we are done rehashing, DELETED marks previously FULL slots. + + // Swap i and new_i elements. + (*transfer)(set, tmp_space, new_slot_ptr); + (*transfer)(set, new_slot_ptr, slot_ptr); + (*transfer)(set, slot_ptr, tmp_space); + + // repeat the processing of the ith slot + --i; + slot_ptr = PrevSlot(slot_ptr, slot_size); + } + } + ResetGrowthLeft(common); + common.infoz().RecordRehash(total_probe_length); +} + +void EraseMetaOnly(CommonFields& c, ctrl_t* it, size_t slot_size) { + assert(IsFull(*it) && "erasing a dangling iterator"); + --c.size_; + const auto index = static_cast<size_t>(it - c.control_); + const size_t index_before = (index - Group::kWidth) & c.capacity_; + const auto empty_after = Group(it).MaskEmpty(); + const auto empty_before = Group(c.control_ + index_before).MaskEmpty(); + + // We count how many consecutive non empties we have to the right and to the + // left of `it`. If the sum is >= kWidth then there is at least one probe + // window that might have seen a full group. + bool was_never_full = empty_before && empty_after && + static_cast<size_t>(empty_after.TrailingZeros()) + + empty_before.LeadingZeros() < + Group::kWidth; + + SetCtrl(c, index, was_never_full ? ctrl_t::kEmpty : ctrl_t::kDeleted, + slot_size); + c.growth_left() += (was_never_full ? 1 : 0); + c.infoz().RecordErase(); +} + +void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy, + bool reuse) { + c.size_ = 0; + if (reuse) { + ResetCtrl(c, policy.slot_size); + c.infoz().RecordStorageChanged(0, c.capacity_); + } else { + void* set = &c; + (*policy.dealloc)(set, policy, c.control_, c.slots_, c.capacity_); + c.control_ = EmptyGroup(); + c.set_generation_ptr(EmptyGeneration()); + c.slots_ = nullptr; + c.capacity_ = 0; + c.growth_left() = 0; + c.infoz().RecordClearedReservation(); + assert(c.size_ == 0); + c.infoz().RecordStorageChanged(0, 0); + } +} } // namespace container_internal Y_ABSL_NAMESPACE_END diff --git a/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/raw_hash_set.h b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/raw_hash_set.h index e24a2bb813..26fda8b83c 100644 --- a/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/raw_hash_set.h +++ b/contrib/restricted/abseil-cpp-tstring/y_absl/container/internal/raw_hash_set.h @@ -186,6 +186,7 @@ #include "y_absl/base/config.h" #include "y_absl/base/internal/endian.h" #include "y_absl/base/internal/prefetch.h" +#include "y_absl/base/internal/raw_logging.h" #include "y_absl/base/optimization.h" #include "y_absl/base/port.h" #include "y_absl/container/internal/common.h" @@ -219,6 +220,29 @@ namespace y_absl { Y_ABSL_NAMESPACE_BEGIN namespace container_internal { +#ifdef Y_ABSL_SWISSTABLE_ENABLE_GENERATIONS +#error Y_ABSL_SWISSTABLE_ENABLE_GENERATIONS cannot be directly set +#elif defined(Y_ABSL_HAVE_ADDRESS_SANITIZER) || \ + defined(Y_ABSL_HAVE_MEMORY_SANITIZER) +// When compiled in sanitizer mode, we add generation integers to the backing +// array and iterators. In the backing array, we store the generation between +// the control bytes and the slots. When iterators are dereferenced, we assert +// that the container has not been mutated in a way that could cause iterator +// invalidation since the iterator was initialized. +#define Y_ABSL_SWISSTABLE_ENABLE_GENERATIONS +#endif + +// We use uint8_t so we don't need to worry about padding. +using GenerationType = uint8_t; + +#ifdef Y_ABSL_SWISSTABLE_ENABLE_GENERATIONS +constexpr bool SwisstableGenerationsEnabled() { return true; } +constexpr size_t NumGenerationBytes() { return sizeof(GenerationType); } +#else +constexpr bool SwisstableGenerationsEnabled() { return false; } +constexpr size_t NumGenerationBytes() { return 0; } +#endif + template <typename AllocType> void SwapAlloc(AllocType& lhs, AllocType& rhs, std::true_type /* propagate_on_container_swap */) { @@ -451,7 +475,7 @@ static_assert(ctrl_t::kDeleted == static_cast<ctrl_t>(-2), "ctrl_t::kDeleted must be -2 to make the implementation of " "ConvertSpecialToEmptyAndFullToDeleted efficient"); -Y_ABSL_DLL extern const ctrl_t kEmptyGroup[16]; +Y_ABSL_DLL extern const ctrl_t kEmptyGroup[17]; // Returns a pointer to a control byte group that can be used by empty tables. inline ctrl_t* EmptyGroup() { @@ -460,6 +484,12 @@ inline ctrl_t* EmptyGroup() { return const_cast<ctrl_t*>(kEmptyGroup); } +// Returns a pointer to the generation byte at the end of the empty group, if it +// exists. +inline GenerationType* EmptyGeneration() { + return reinterpret_cast<GenerationType*>(EmptyGroup() + 16); +} + // Mixes a randomly generated per-process seed with `hash` and `ctrl` to // randomize insertion order within groups. bool ShouldInsertBackwards(size_t hash, const ctrl_t* ctrl); @@ -545,7 +575,7 @@ struct GroupSse2Impl { // Returns a bitmask representing the positions of slots that match hash. BitMask<uint32_t, kWidth> Match(h2_t hash) const { - auto match = _mm_set1_epi8(hash); + auto match = _mm_set1_epi8(static_cast<char>(hash)); return BitMask<uint32_t, kWidth>( static_cast<uint32_t>(_mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl)))); } @@ -557,7 +587,7 @@ struct GroupSse2Impl { return NonIterableBitMask<uint32_t, kWidth>( static_cast<uint32_t>(_mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl)))); #else - auto match = _mm_set1_epi8(static_cast<h2_t>(ctrl_t::kEmpty)); + auto match = _mm_set1_epi8(static_cast<char>(ctrl_t::kEmpty)); return NonIterableBitMask<uint32_t, kWidth>( static_cast<uint32_t>(_mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl)))); #endif @@ -565,14 +595,14 @@ struct GroupSse2Impl { // Returns a bitmask representing the positions of empty or deleted slots. NonIterableBitMask<uint32_t, kWidth> MaskEmptyOrDeleted() const { - auto special = _mm_set1_epi8(static_cast<uint8_t>(ctrl_t::kSentinel)); + auto special = _mm_set1_epi8(static_cast<char>(ctrl_t::kSentinel)); return NonIterableBitMask<uint32_t, kWidth>(static_cast<uint32_t>( _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)))); } // Returns the number of trailing empty or deleted elements in the group. uint32_t CountLeadingEmptyOrDeleted() const { - auto special = _mm_set1_epi8(static_cast<uint8_t>(ctrl_t::kSentinel)); + auto special = _mm_set1_epi8(static_cast<char>(ctrl_t::kSentinel)); return TrailingZeros(static_cast<uint32_t>( _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)) + 1)); } @@ -612,9 +642,9 @@ struct GroupAArch64Impl { NonIterableBitMask<uint64_t, kWidth, 3> MaskEmpty() const { uint64_t mask = - vget_lane_u64(vreinterpret_u64_u8( - vceq_s8(vdup_n_s8(static_cast<h2_t>(ctrl_t::kEmpty)), - vreinterpret_s8_u8(ctrl))), + vget_lane_u64(vreinterpret_u64_u8(vceq_s8( + vdup_n_s8(static_cast<int8_t>(ctrl_t::kEmpty)), + vreinterpret_s8_u8(ctrl))), 0); return NonIterableBitMask<uint64_t, kWidth, 3>(mask); } @@ -629,13 +659,16 @@ struct GroupAArch64Impl { } uint32_t CountLeadingEmptyOrDeleted() const { - uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(ctrl), 0); - // ctrl | ~(ctrl >> 7) will have the lowest bit set to zero for kEmpty and - // kDeleted. We lower all other bits and count number of trailing zeros. + uint64_t mask = + vget_lane_u64(vreinterpret_u64_u8(vcle_s8( + vdup_n_s8(static_cast<int8_t>(ctrl_t::kSentinel)), + vreinterpret_s8_u8(ctrl))), + 0); + // Similar to MaskEmptyorDeleted() but we invert the logic to invert the + // produced bitfield. We then count number of trailing zeros. // Clang and GCC optimize countr_zero to rbit+clz without any check for 0, // so we should be fine. - constexpr uint64_t bits = 0x0101010101010101ULL; - return countr_zero((mask | ~(mask >> 7)) & bits) >> 3; + return static_cast<uint32_t>(countr_zero(mask)) >> 3; } void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { @@ -693,7 +726,8 @@ struct GroupPortableImpl { // ctrl | ~(ctrl >> 7) will have the lowest bit set to zero for kEmpty and // kDeleted. We lower all other bits and count number of trailing zeros. constexpr uint64_t bits = 0x0101010101010101ULL; - return countr_zero((ctrl | ~(ctrl >> 7)) & bits) >> 3; + return static_cast<uint32_t>(countr_zero((ctrl | ~(ctrl >> 7)) & bits) >> + 3); } void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { @@ -715,6 +749,192 @@ using Group = GroupAArch64Impl; using Group = GroupPortableImpl; #endif +class CommonFieldsGenerationInfoEnabled { + // A sentinel value for reserved_growth_ indicating that we just ran out of + // reserved growth on the last insertion. When reserve is called and then + // insertions take place, reserved_growth_'s state machine is N, ..., 1, + // kReservedGrowthJustRanOut, 0. + static constexpr size_t kReservedGrowthJustRanOut = + (std::numeric_limits<size_t>::max)(); + + public: + CommonFieldsGenerationInfoEnabled() = default; + CommonFieldsGenerationInfoEnabled(CommonFieldsGenerationInfoEnabled&& that) + : reserved_growth_(that.reserved_growth_), generation_(that.generation_) { + that.reserved_growth_ = 0; + that.generation_ = EmptyGeneration(); + } + CommonFieldsGenerationInfoEnabled& operator=( + CommonFieldsGenerationInfoEnabled&&) = default; + + // Whether we should rehash on insert in order to detect bugs of using invalid + // references. We rehash on the first insertion after reserved_growth_ reaches + // 0 after a call to reserve. + // TODO(b/254649633): we could potentially do a rehash with low probability + // whenever reserved_growth_ is zero. + bool should_rehash_for_bug_detection_on_insert() const { + return reserved_growth_ == kReservedGrowthJustRanOut; + } + void maybe_increment_generation_on_insert() { + if (reserved_growth_ == kReservedGrowthJustRanOut) reserved_growth_ = 0; + + if (reserved_growth_ > 0) { + if (--reserved_growth_ == 0) reserved_growth_ = kReservedGrowthJustRanOut; + } else { + ++*generation_; + } + } + void reset_reserved_growth(size_t reservation, size_t size) { + reserved_growth_ = reservation - size; + } + size_t reserved_growth() const { return reserved_growth_; } + void set_reserved_growth(size_t r) { reserved_growth_ = r; } + GenerationType generation() const { return *generation_; } + void set_generation(GenerationType g) { *generation_ = g; } + GenerationType* generation_ptr() const { return generation_; } + void set_generation_ptr(GenerationType* g) { generation_ = g; } + + private: + // The number of insertions remaining that are guaranteed to not rehash due to + // a prior call to reserve. Note: we store reserved growth rather than + // reservation size because calls to erase() decrease size_ but don't decrease + // reserved growth. + size_t reserved_growth_ = 0; + // Pointer to the generation counter, which is used to validate iterators and + // is stored in the backing array between the control bytes and the slots. + // Note that we can't store the generation inside the container itself and + // keep a pointer to the container in the iterators because iterators must + // remain valid when the container is moved. + // Note: we could derive this pointer from the control pointer, but it makes + // the code more complicated, and there's a benefit in having the sizes of + // raw_hash_set in sanitizer mode and non-sanitizer mode a bit more different, + // which is that tests are less likely to rely on the size remaining the same. + GenerationType* generation_ = EmptyGeneration(); +}; + +class CommonFieldsGenerationInfoDisabled { + public: + CommonFieldsGenerationInfoDisabled() = default; + CommonFieldsGenerationInfoDisabled(CommonFieldsGenerationInfoDisabled&&) = + default; + CommonFieldsGenerationInfoDisabled& operator=( + CommonFieldsGenerationInfoDisabled&&) = default; + + bool should_rehash_for_bug_detection_on_insert() const { return false; } + void maybe_increment_generation_on_insert() {} + void reset_reserved_growth(size_t, size_t) {} + size_t reserved_growth() const { return 0; } + void set_reserved_growth(size_t) {} + GenerationType generation() const { return 0; } + void set_generation(GenerationType) {} + GenerationType* generation_ptr() const { return nullptr; } + void set_generation_ptr(GenerationType*) {} +}; + +class HashSetIteratorGenerationInfoEnabled { + public: + HashSetIteratorGenerationInfoEnabled() = default; + explicit HashSetIteratorGenerationInfoEnabled( + const GenerationType* generation_ptr) + : generation_ptr_(generation_ptr), generation_(*generation_ptr) {} + + GenerationType generation() const { return generation_; } + void reset_generation() { generation_ = *generation_ptr_; } + const GenerationType* generation_ptr() const { return generation_ptr_; } + void set_generation_ptr(const GenerationType* ptr) { generation_ptr_ = ptr; } + + private: + const GenerationType* generation_ptr_ = EmptyGeneration(); + GenerationType generation_ = *generation_ptr_; +}; + +class HashSetIteratorGenerationInfoDisabled { + public: + HashSetIteratorGenerationInfoDisabled() = default; + explicit HashSetIteratorGenerationInfoDisabled(const GenerationType*) {} + + GenerationType generation() const { return 0; } + void reset_generation() {} + const GenerationType* generation_ptr() const { return nullptr; } + void set_generation_ptr(const GenerationType*) {} +}; + +#ifdef Y_ABSL_SWISSTABLE_ENABLE_GENERATIONS +using CommonFieldsGenerationInfo = CommonFieldsGenerationInfoEnabled; +using HashSetIteratorGenerationInfo = HashSetIteratorGenerationInfoEnabled; +#else +using CommonFieldsGenerationInfo = CommonFieldsGenerationInfoDisabled; +using HashSetIteratorGenerationInfo = HashSetIteratorGenerationInfoDisabled; +#endif + +// CommonFields hold the fields in raw_hash_set that do not depend +// on template parameters. This allows us to conveniently pass all +// of this state to helper functions as a single argument. +class CommonFields : public CommonFieldsGenerationInfo { + public: + CommonFields() = default; + + // Not copyable + CommonFields(const CommonFields&) = delete; + CommonFields& operator=(const CommonFields&) = delete; + + // Movable + CommonFields(CommonFields&& that) + : CommonFieldsGenerationInfo( + std::move(static_cast<CommonFieldsGenerationInfo&&>(that))), + // Explicitly copying fields into "this" and then resetting "that" + // fields generates less code then calling y_absl::exchange per field. + control_(that.control_), + slots_(that.slots_), + size_(that.size_), + capacity_(that.capacity_), + compressed_tuple_(that.growth_left(), std::move(that.infoz())) { + that.control_ = EmptyGroup(); + that.slots_ = nullptr; + that.size_ = 0; + that.capacity_ = 0; + that.growth_left() = 0; + } + CommonFields& operator=(CommonFields&&) = default; + + // The number of slots we can still fill without needing to rehash. + size_t& growth_left() { return compressed_tuple_.template get<0>(); } + + HashtablezInfoHandle& infoz() { return compressed_tuple_.template get<1>(); } + const HashtablezInfoHandle& infoz() const { + return compressed_tuple_.template get<1>(); + } + + void reset_reserved_growth(size_t reservation) { + CommonFieldsGenerationInfo::reset_reserved_growth(reservation, size_); + } + + // TODO(b/259599413): Investigate removing some of these fields: + // - control/slots can be derived from each other + // - size can be moved into the slot array + + // The control bytes (and, also, a pointer to the base of the backing array). + // + // This contains `capacity + 1 + NumClonedBytes()` entries, even + // when the table is empty (hence EmptyGroup). + ctrl_t* control_ = EmptyGroup(); + + // The beginning of the slots, located at `SlotOffset()` bytes after + // `control`. May be null for empty tables. + void* slots_ = nullptr; + + // The number of filled slots. + size_t size_ = 0; + + // The total number of available slots. + size_t capacity_ = 0; + + // Bundle together growth_left and HashtablezInfoHandle to ensure EBO for + // HashtablezInfoHandle when sampling is turned off. + y_absl::container_internal::CompressedTuple<size_t, HashtablezInfoHandle> + compressed_tuple_{0u, HashtablezInfoHandle{}}; +}; + // Returns he number of "cloned control bytes". // // This is the number of control bytes that are present both at the beginning @@ -730,6 +950,12 @@ class raw_hash_set; // A valid capacity is a non-zero integer `2^m - 1`. inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; } +// Returns the next valid capacity after `n`. +inline size_t NextCapacity(size_t n) { + assert(IsValidCapacity(n) || n == 0); + return n * 2 + 1; +} + // Applies the following mapping to every byte in the control array: // * kDeleted -> kEmpty // * kEmpty -> kEmpty @@ -795,15 +1021,69 @@ size_t SelectBucketCountForIterRange(InputIter first, InputIter last, return 0; } -#define Y_ABSL_INTERNAL_ASSERT_IS_FULL(ctrl, msg) \ - Y_ABSL_HARDENING_ASSERT((ctrl != nullptr && IsFull(*ctrl)) && msg) +#define Y_ABSL_INTERNAL_ASSERT_IS_FULL(ctrl, generation, generation_ptr, \ + operation) \ + do { \ + Y_ABSL_HARDENING_ASSERT( \ + (ctrl != nullptr) && operation \ + " called on invalid iterator. The iterator might be an end() " \ + "iterator or may have been default constructed."); \ + if (SwisstableGenerationsEnabled() && generation != *generation_ptr) \ + Y_ABSL_INTERNAL_LOG(FATAL, operation \ + " called on invalidated iterator. The table could " \ + "have rehashed since this iterator was initialized."); \ + Y_ABSL_HARDENING_ASSERT( \ + (IsFull(*ctrl)) && operation \ + " called on invalid iterator. The element might have been erased or " \ + "the table might have rehashed."); \ + } while (0) + +// Note that for comparisons, null/end iterators are valid. +inline void AssertIsValidForComparison(const ctrl_t* ctrl, + GenerationType generation, + const GenerationType* generation_ptr) { + Y_ABSL_HARDENING_ASSERT((ctrl == nullptr || IsFull(*ctrl)) && + "Invalid iterator comparison. The element might have " + "been erased or the table might have rehashed."); + if (SwisstableGenerationsEnabled() && generation != *generation_ptr) { + Y_ABSL_INTERNAL_LOG(FATAL, + "Invalid iterator comparison. The table could have " + "rehashed since this iterator was initialized."); + } +} + +// If the two iterators come from the same container, then their pointers will +// interleave such that ctrl_a <= ctrl_b < slot_a <= slot_b or vice/versa. +// Note: we take slots by reference so that it's not UB if they're uninitialized +// as long as we don't read them (when ctrl is null). +inline bool AreItersFromSameContainer(const ctrl_t* ctrl_a, + const ctrl_t* ctrl_b, + const void* const& slot_a, + const void* const& slot_b) { + // If either control byte is null, then we can't tell. + if (ctrl_a == nullptr || ctrl_b == nullptr) return true; + const void* low_slot = slot_a; + const void* hi_slot = slot_b; + if (ctrl_a > ctrl_b) { + std::swap(ctrl_a, ctrl_b); + std::swap(low_slot, hi_slot); + } + return ctrl_b < low_slot && low_slot <= hi_slot; +} -inline void AssertIsValid(ctrl_t* ctrl) { +// Asserts that two iterators come from the same container. +// Note: we take slots by reference so that it's not UB if they're uninitialized +// as long as we don't read them (when ctrl is null). +// TODO(b/254649633): when generations are enabled, we can detect more cases of +// different containers by comparing the pointers to the generations - this +// can cover cases of end iterators that we would otherwise miss. +inline void AssertSameContainer(const ctrl_t* ctrl_a, const ctrl_t* ctrl_b, + const void* const& slot_a, + const void* const& slot_b) { Y_ABSL_HARDENING_ASSERT( - (ctrl == nullptr || IsFull(*ctrl)) && - "Invalid operation on iterator. The element might have " - "been erased, the table might have rehashed, or this may " - "be an end() iterator."); + AreItersFromSameContainer(ctrl_a, ctrl_b, slot_a, slot_b) && + "Invalid iterator comparison. The iterators may be from different " + "containers or the container might have rehashed."); } struct FindInfo { @@ -825,9 +1105,10 @@ struct FindInfo { // `ShouldInsertBackwards()` for small tables. inline bool is_small(size_t capacity) { return capacity < Group::kWidth - 1; } -// Begins a probing operation on `ctrl`, using `hash`. -inline probe_seq<Group::kWidth> probe(const ctrl_t* ctrl, size_t hash, - size_t capacity) { +// Begins a probing operation on `common.control`, using `hash`. +inline probe_seq<Group::kWidth> probe(const CommonFields& common, size_t hash) { + const ctrl_t* ctrl = common.control_; + const size_t capacity = common.capacity_; return probe_seq<Group::kWidth>(H1(hash, ctrl), capacity); } @@ -839,9 +1120,9 @@ inline probe_seq<Group::kWidth> probe(const ctrl_t* ctrl, size_t hash, // NOTE: this function must work with tables having both empty and deleted // slots in the same group. Such tables appear during `erase()`. template <typename = void> -inline FindInfo find_first_non_full(const ctrl_t* ctrl, size_t hash, - size_t capacity) { - auto seq = probe(ctrl, hash, capacity); +inline FindInfo find_first_non_full(const CommonFields& common, size_t hash) { + auto seq = probe(common, hash); + const ctrl_t* ctrl = common.control_; while (true) { Group g{ctrl + seq.offset()}; auto mask = g.MaskEmptyOrDeleted(); @@ -851,55 +1132,75 @@ inline FindInfo find_first_non_full(const ctrl_t* ctrl, size_t hash, // In debug build we will randomly insert in either the front or back of // the group. // TODO(kfm,sbenza): revisit after we do unconditional mixing - if (!is_small(capacity) && ShouldInsertBackwards(hash, ctrl)) { + if (!is_small(common.capacity_) && ShouldInsertBackwards(hash, ctrl)) { return {seq.offset(mask.HighestBitSet()), seq.index()}; } #endif return {seq.offset(mask.LowestBitSet()), seq.index()}; } seq.next(); - assert(seq.index() <= capacity && "full table!"); + assert(seq.index() <= common.capacity_ && "full table!"); } } // Extern template for inline function keep possibility of inlining. // When compiler decided to not inline, no symbols will be added to the // corresponding translation unit. -extern template FindInfo find_first_non_full(const ctrl_t*, size_t, size_t); +extern template FindInfo find_first_non_full(const CommonFields&, size_t); + +// Non-inlined version of find_first_non_full for use in less +// performance critical routines. +FindInfo find_first_non_full_outofline(const CommonFields&, size_t); + +inline void ResetGrowthLeft(CommonFields& common) { + common.growth_left() = CapacityToGrowth(common.capacity_) - common.size_; +} // Sets `ctrl` to `{kEmpty, kSentinel, ..., kEmpty}`, marking the entire // array as marked as empty. -inline void ResetCtrl(size_t capacity, ctrl_t* ctrl, const void* slot, - size_t slot_size) { +inline void ResetCtrl(CommonFields& common, size_t slot_size) { + const size_t capacity = common.capacity_; + ctrl_t* ctrl = common.control_; std::memset(ctrl, static_cast<int8_t>(ctrl_t::kEmpty), capacity + 1 + NumClonedBytes()); ctrl[capacity] = ctrl_t::kSentinel; - SanitizerPoisonMemoryRegion(slot, slot_size * capacity); + SanitizerPoisonMemoryRegion(common.slots_, slot_size * capacity); + ResetGrowthLeft(common); } // Sets `ctrl[i]` to `h`. // // Unlike setting it directly, this function will perform bounds checks and // mirror the value to the cloned tail if necessary. -inline void SetCtrl(size_t i, ctrl_t h, size_t capacity, ctrl_t* ctrl, - const void* slot, size_t slot_size) { +inline void SetCtrl(const CommonFields& common, size_t i, ctrl_t h, + size_t slot_size) { + const size_t capacity = common.capacity_; assert(i < capacity); - auto* slot_i = static_cast<const char*>(slot) + i * slot_size; + auto* slot_i = static_cast<const char*>(common.slots_) + i * slot_size; if (IsFull(h)) { SanitizerUnpoisonMemoryRegion(slot_i, slot_size); } else { SanitizerPoisonMemoryRegion(slot_i, slot_size); } + ctrl_t* ctrl = common.control_; ctrl[i] = h; ctrl[((i - NumClonedBytes()) & capacity) + (NumClonedBytes() & capacity)] = h; } // Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`. -inline void SetCtrl(size_t i, h2_t h, size_t capacity, ctrl_t* ctrl, - const void* slot, size_t slot_size) { - SetCtrl(i, static_cast<ctrl_t>(h), capacity, ctrl, slot, slot_size); +inline void SetCtrl(const CommonFields& common, size_t i, h2_t h, + size_t slot_size) { + SetCtrl(common, i, static_cast<ctrl_t>(h), slot_size); +} + +// Given the capacity of a table, computes the offset (from the start of the +// backing allocation) of the generation counter (if it exists). +inline size_t GenerationOffset(size_t capacity) { + assert(IsValidCapacity(capacity)); + const size_t num_control_bytes = capacity + 1 + NumClonedBytes(); + return num_control_bytes; } // Given the capacity of a table, computes the offset (from the start of the @@ -907,7 +1208,8 @@ inline void SetCtrl(size_t i, h2_t h, size_t capacity, ctrl_t* ctrl, inline size_t SlotOffset(size_t capacity, size_t slot_align) { assert(IsValidCapacity(capacity)); const size_t num_control_bytes = capacity + 1 + NumClonedBytes(); - return (num_control_bytes + slot_align - 1) & (~slot_align + 1); + return (num_control_bytes + NumGenerationBytes() + slot_align - 1) & + (~slot_align + 1); } // Given the capacity of a table, computes the total size of the backing @@ -916,6 +1218,91 @@ inline size_t AllocSize(size_t capacity, size_t slot_size, size_t slot_align) { return SlotOffset(capacity, slot_align) + capacity * slot_size; } +template <typename Alloc, size_t SizeOfSlot, size_t AlignOfSlot> +Y_ABSL_ATTRIBUTE_NOINLINE void InitializeSlots(CommonFields& c, Alloc alloc) { + assert(c.capacity_); + // Folks with custom allocators often make unwarranted assumptions about the + // behavior of their classes vis-a-vis trivial destructability and what + // calls they will or won't make. Avoid sampling for people with custom + // allocators to get us out of this mess. This is not a hard guarantee but + // a workaround while we plan the exact guarantee we want to provide. + const size_t sample_size = + (std::is_same<Alloc, std::allocator<char>>::value && c.slots_ == nullptr) + ? SizeOfSlot + : 0; + + const size_t cap = c.capacity_; + char* mem = static_cast<char*>( + Allocate<AlignOfSlot>(&alloc, AllocSize(cap, SizeOfSlot, AlignOfSlot))); + const GenerationType old_generation = c.generation(); + c.set_generation_ptr( + reinterpret_cast<GenerationType*>(mem + GenerationOffset(cap))); + c.set_generation(old_generation + 1); + c.control_ = reinterpret_cast<ctrl_t*>(mem); + c.slots_ = mem + SlotOffset(cap, AlignOfSlot); + ResetCtrl(c, SizeOfSlot); + if (sample_size) { + c.infoz() = Sample(sample_size); + } + c.infoz().RecordStorageChanged(c.size_, cap); +} + +// PolicyFunctions bundles together some information for a particular +// raw_hash_set<T, ...> instantiation. This information is passed to +// type-erased functions that want to do small amounts of type-specific +// work. +struct PolicyFunctions { + size_t slot_size; + + // Return the hash of the pointed-to slot. + size_t (*hash_slot)(void* set, void* slot); + + // Transfer the contents of src_slot to dst_slot. + void (*transfer)(void* set, void* dst_slot, void* src_slot); + + // Deallocate the specified backing store which is sized for n slots. + void (*dealloc)(void* set, const PolicyFunctions& policy, ctrl_t* ctrl, + void* slot_array, size_t n); +}; + +// ClearBackingArray clears the backing array, either modifying it in place, +// or creating a new one based on the value of "reuse". +// REQUIRES: c.capacity > 0 +void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy, + bool reuse); + +// Type-erased version of raw_hash_set::erase_meta_only. +void EraseMetaOnly(CommonFields& c, ctrl_t* it, size_t slot_size); + +// Function to place in PolicyFunctions::dealloc for raw_hash_sets +// that are using std::allocator. This allows us to share the same +// function body for raw_hash_set instantiations that have the +// same slot alignment. +template <size_t AlignOfSlot> +Y_ABSL_ATTRIBUTE_NOINLINE void DeallocateStandard(void*, + const PolicyFunctions& policy, + ctrl_t* ctrl, void* slot_array, + size_t n) { + // Unpoison before returning the memory to the allocator. + SanitizerUnpoisonMemoryRegion(slot_array, policy.slot_size * n); + + std::allocator<char> alloc; + Deallocate<AlignOfSlot>(&alloc, ctrl, + AllocSize(n, policy.slot_size, AlignOfSlot)); +} + +// For trivially relocatable types we use memcpy directly. This allows us to +// share the same function body for raw_hash_set instantiations that have the +// same slot size as long as they are relocatable. +template <size_t SizeOfSlot> +Y_ABSL_ATTRIBUTE_NOINLINE void TransferRelocatable(void*, void* dst, void* src) { + memcpy(dst, src, SizeOfSlot); +} + +// Type-erased version of raw_hash_set::drop_deletes_without_resize. +void DropDeletesWithoutResize(CommonFields& common, + const PolicyFunctions& policy, void* tmp_space); + // A SwissTable. // // Policy: a policy defines how to perform different operations on @@ -1016,7 +1403,7 @@ class raw_hash_set { static_assert(std::is_same<const_pointer, const value_type*>::value, "Allocators with custom pointer types are not supported"); - class iterator { + class iterator : private HashSetIteratorGenerationInfo { friend class raw_hash_set; public: @@ -1032,22 +1419,22 @@ class raw_hash_set { // PRECONDITION: not an end() iterator. reference operator*() const { - Y_ABSL_INTERNAL_ASSERT_IS_FULL(ctrl_, - "operator*() called on invalid iterator."); + Y_ABSL_INTERNAL_ASSERT_IS_FULL(ctrl_, generation(), generation_ptr(), + "operator*()"); return PolicyTraits::element(slot_); } // PRECONDITION: not an end() iterator. pointer operator->() const { - Y_ABSL_INTERNAL_ASSERT_IS_FULL(ctrl_, - "operator-> called on invalid iterator."); + Y_ABSL_INTERNAL_ASSERT_IS_FULL(ctrl_, generation(), generation_ptr(), + "operator->"); return &operator*(); } // PRECONDITION: not an end() iterator. iterator& operator++() { - Y_ABSL_INTERNAL_ASSERT_IS_FULL(ctrl_, - "operator++ called on invalid iterator."); + Y_ABSL_INTERNAL_ASSERT_IS_FULL(ctrl_, generation(), generation_ptr(), + "operator++"); ++ctrl_; ++slot_; skip_empty_or_deleted(); @@ -1061,8 +1448,9 @@ class raw_hash_set { } friend bool operator==(const iterator& a, const iterator& b) { - AssertIsValid(a.ctrl_); - AssertIsValid(b.ctrl_); + AssertSameContainer(a.ctrl_, b.ctrl_, a.slot_, b.slot_); + AssertIsValidForComparison(a.ctrl_, a.generation(), a.generation_ptr()); + AssertIsValidForComparison(b.ctrl_, b.generation(), b.generation_ptr()); return a.ctrl_ == b.ctrl_; } friend bool operator!=(const iterator& a, const iterator& b) { @@ -1070,16 +1458,23 @@ class raw_hash_set { } private: - iterator(ctrl_t* ctrl, slot_type* slot) : ctrl_(ctrl), slot_(slot) { + iterator(ctrl_t* ctrl, slot_type* slot, + const GenerationType* generation_ptr) + : HashSetIteratorGenerationInfo(generation_ptr), + ctrl_(ctrl), + slot_(slot) { // This assumption helps the compiler know that any non-end iterator is // not equal to any end iterator. Y_ABSL_ASSUME(ctrl != nullptr); } + // For end() iterators. + explicit iterator(const GenerationType* generation_ptr) + : HashSetIteratorGenerationInfo(generation_ptr) {} // Fixes up `ctrl_` to point to a full by advancing it and `slot_` until // they reach one. // - // If a sentinel is reached, we null both of them out instead. + // If a sentinel is reached, we null `ctrl_` out instead. void skip_empty_or_deleted() { while (IsEmptyOrDeleted(*ctrl_)) { uint32_t shift = Group{ctrl_}.CountLeadingEmptyOrDeleted(); @@ -1107,9 +1502,9 @@ class raw_hash_set { using pointer = typename raw_hash_set::const_pointer; using difference_type = typename raw_hash_set::difference_type; - const_iterator() {} + const_iterator() = default; // Implicit construction from iterator. - const_iterator(iterator i) : inner_(std::move(i)) {} + const_iterator(iterator i) : inner_(std::move(i)) {} // NOLINT reference operator*() const { return *inner_; } pointer operator->() const { return inner_.operator->(); } @@ -1128,8 +1523,10 @@ class raw_hash_set { } private: - const_iterator(const ctrl_t* ctrl, const slot_type* slot) - : inner_(const_cast<ctrl_t*>(ctrl), const_cast<slot_type*>(slot)) {} + const_iterator(const ctrl_t* ctrl, const slot_type* slot, + const GenerationType* gen) + : inner_(const_cast<ctrl_t*>(ctrl), const_cast<slot_type*>(slot), gen) { + } iterator inner_; }; @@ -1137,18 +1534,20 @@ class raw_hash_set { using node_type = node_handle<Policy, hash_policy_traits<Policy>, Alloc>; using insert_return_type = InsertReturnType<iterator, node_type>; + // Note: can't use `= default` due to non-default noexcept (causes + // problems for some compilers). NOLINTNEXTLINE raw_hash_set() noexcept( std::is_nothrow_default_constructible<hasher>::value&& std::is_nothrow_default_constructible<key_equal>::value&& std::is_nothrow_default_constructible<allocator_type>::value) {} - explicit raw_hash_set(size_t bucket_count, const hasher& hash = hasher(), - const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : ctrl_(EmptyGroup()), - settings_(0, HashtablezInfoHandle(), hash, eq, alloc) { + Y_ABSL_ATTRIBUTE_NOINLINE explicit raw_hash_set( + size_t bucket_count, const hasher& hash = hasher(), + const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : settings_(CommonFields{}, hash, eq, alloc) { if (bucket_count) { - capacity_ = NormalizeCapacity(bucket_count); + common().capacity_ = NormalizeCapacity(bucket_count); initialize_slots(); } } @@ -1255,45 +1654,30 @@ class raw_hash_set { // than a full `insert`. for (const auto& v : that) { const size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, v); - auto target = find_first_non_full(ctrl_, hash, capacity_); - SetCtrl(target.offset, H2(hash), capacity_, ctrl_, slots_, - sizeof(slot_type)); + auto target = find_first_non_full_outofline(common(), hash); + SetCtrl(common(), target.offset, H2(hash), sizeof(slot_type)); emplace_at(target.offset, v); + common().maybe_increment_generation_on_insert(); infoz().RecordInsert(hash, target.probe_length); } - size_ = that.size(); + common().size_ = that.size(); growth_left() -= that.size(); } - raw_hash_set(raw_hash_set&& that) noexcept( + Y_ABSL_ATTRIBUTE_NOINLINE raw_hash_set(raw_hash_set&& that) noexcept( std::is_nothrow_copy_constructible<hasher>::value&& std::is_nothrow_copy_constructible<key_equal>::value&& std::is_nothrow_copy_constructible<allocator_type>::value) - : ctrl_(y_absl::exchange(that.ctrl_, EmptyGroup())), - slots_(y_absl::exchange(that.slots_, nullptr)), - size_(y_absl::exchange(that.size_, 0)), - capacity_(y_absl::exchange(that.capacity_, 0)), - // Hash, equality and allocator are copied instead of moved because - // `that` must be left valid. If Hash is std::function<Key>, moving it - // would create a nullptr functor that cannot be called. - settings_(y_absl::exchange(that.growth_left(), 0), - y_absl::exchange(that.infoz(), HashtablezInfoHandle()), + : // Hash, equality and allocator are copied instead of moved because + // `that` must be left valid. If Hash is std::function<Key>, moving it + // would create a nullptr functor that cannot be called. + settings_(y_absl::exchange(that.common(), CommonFields{}), that.hash_ref(), that.eq_ref(), that.alloc_ref()) {} raw_hash_set(raw_hash_set&& that, const allocator_type& a) - : ctrl_(EmptyGroup()), - slots_(nullptr), - size_(0), - capacity_(0), - settings_(0, HashtablezInfoHandle(), that.hash_ref(), that.eq_ref(), - a) { + : settings_(CommonFields{}, that.hash_ref(), that.eq_ref(), a) { if (a == that.alloc_ref()) { - std::swap(ctrl_, that.ctrl_); - std::swap(slots_, that.slots_); - std::swap(size_, that.size_); - std::swap(capacity_, that.capacity_); - std::swap(growth_left(), that.growth_left()); - std::swap(infoz(), that.infoz()); + std::swap(common(), that.common()); } else { reserve(that.size()); // Note: this will copy elements of dense_set and unordered_set instead of @@ -1317,30 +1701,43 @@ class raw_hash_set { std::is_nothrow_move_assignable<key_equal>::value) { // TODO(sbenza): We should only use the operations from the noexcept clause // to make sure we actually adhere to that contract. + // NOLINTNEXTLINE: not returning *this for performance. return move_assign( std::move(that), typename AllocTraits::propagate_on_container_move_assignment()); } - ~raw_hash_set() { destroy_slots(); } + ~raw_hash_set() { + const size_t cap = capacity(); + if (!cap) return; + destroy_slots(); + + // Unpoison before returning the memory to the allocator. + SanitizerUnpoisonMemoryRegion(slot_array(), sizeof(slot_type) * cap); + Deallocate<alignof(slot_type)>( + &alloc_ref(), control(), + AllocSize(cap, sizeof(slot_type), alignof(slot_type))); + + infoz().Unregister(); + } iterator begin() { auto it = iterator_at(0); it.skip_empty_or_deleted(); return it; } - iterator end() { return {}; } + iterator end() { return iterator(common().generation_ptr()); } const_iterator begin() const { return const_cast<raw_hash_set*>(this)->begin(); } - const_iterator end() const { return {}; } + const_iterator end() const { return iterator(common().generation_ptr()); } const_iterator cbegin() const { return begin(); } const_iterator cend() const { return end(); } bool empty() const { return !size(); } - size_t size() const { return size_; } - size_t capacity() const { return capacity_; } + size_t size() const { return common().size_; } + size_t capacity() const { return common().capacity_; } size_t max_size() const { return (std::numeric_limits<size_t>::max)(); } Y_ABSL_ATTRIBUTE_REINITIALIZES void clear() { @@ -1351,22 +1748,26 @@ class raw_hash_set { // compared to destruction of the elements of the container. So we pick the // largest bucket_count() threshold for which iteration is still fast and // past that we simply deallocate the array. - if (capacity_ > 127) { + const size_t cap = capacity(); + if (cap == 0) { + // Already guaranteed to be empty; so nothing to do. + } else { destroy_slots(); + ClearBackingArray(common(), GetPolicyFunctions(), + /*reuse=*/cap < 128); + } + common().set_reserved_growth(0); + } - infoz().RecordClearedReservation(); - } else if (capacity_) { - for (size_t i = 0; i != capacity_; ++i) { - if (IsFull(ctrl_[i])) { - PolicyTraits::destroy(&alloc_ref(), slots_ + i); - } + inline void destroy_slots() { + const size_t cap = capacity(); + const ctrl_t* ctrl = control(); + slot_type* slot = slot_array(); + for (size_t i = 0; i != cap; ++i) { + if (IsFull(ctrl[i])) { + PolicyTraits::destroy(&alloc_ref(), slot + i); } - size_ = 0; - ResetCtrl(capacity_, ctrl_, slots_, sizeof(slot_type)); - reset_growth_left(); } - assert(empty()); - infoz().RecordStorageChanged(0, capacity_); } // This overload kicks in when the argument is an rvalue of insertable and @@ -1554,7 +1955,7 @@ class raw_hash_set { iterator lazy_emplace(const key_arg<K>& key, F&& f) { auto res = find_or_prepare_insert(key); if (res.second) { - slot_type* slot = slots_ + res.first; + slot_type* slot = slot_array() + res.first; std::forward<F>(f)(constructor(&alloc_ref(), &slot)); assert(!slot); } @@ -1596,8 +1997,8 @@ class raw_hash_set { // This overload is necessary because otherwise erase<K>(const K&) would be // a better match if non-const iterator is passed as an argument. void erase(iterator it) { - Y_ABSL_INTERNAL_ASSERT_IS_FULL(it.ctrl_, - "erase() called on invalid iterator."); + Y_ABSL_INTERNAL_ASSERT_IS_FULL(it.ctrl_, it.generation(), it.generation_ptr(), + "erase()"); PolicyTraits::destroy(&alloc_ref(), it.slot_); erase_meta_only(it); } @@ -1632,7 +2033,8 @@ class raw_hash_set { node_type extract(const_iterator position) { Y_ABSL_INTERNAL_ASSERT_IS_FULL(position.inner_.ctrl_, - "extract() called on invalid iterator."); + position.inner_.generation(), + position.inner_.generation_ptr(), "extract()"); auto node = CommonAccess::Transfer<node_type>(alloc_ref(), position.inner_.slot_); erase_meta_only(position); @@ -1652,24 +2054,18 @@ class raw_hash_set { IsNoThrowSwappable<allocator_type>( typename AllocTraits::propagate_on_container_swap{})) { using std::swap; - swap(ctrl_, that.ctrl_); - swap(slots_, that.slots_); - swap(size_, that.size_); - swap(capacity_, that.capacity_); - swap(growth_left(), that.growth_left()); + swap(common(), that.common()); swap(hash_ref(), that.hash_ref()); swap(eq_ref(), that.eq_ref()); - swap(infoz(), that.infoz()); SwapAlloc(alloc_ref(), that.alloc_ref(), typename AllocTraits::propagate_on_container_swap{}); } void rehash(size_t n) { - if (n == 0 && capacity_ == 0) return; - if (n == 0 && size_ == 0) { - destroy_slots(); - infoz().RecordStorageChanged(0, 0); - infoz().RecordClearedReservation(); + if (n == 0 && capacity() == 0) return; + if (n == 0 && size() == 0) { + ClearBackingArray(common(), GetPolicyFunctions(), + /*reuse=*/false); return; } @@ -1677,7 +2073,7 @@ class raw_hash_set { // power-of-2-minus-1, so bitor is good enough. auto m = NormalizeCapacity(n | GrowthToLowerboundCapacity(size())); // n == 0 unconditionally rehashes as per the standard. - if (n == 0 || m > capacity_) { + if (n == 0 || m > capacity()) { resize(m); // This is after resize, to ensure that we have completed the allocation @@ -1695,6 +2091,7 @@ class raw_hash_set { // and have potentially sampled the hashtable. infoz().RecordReservation(n); } + common().reset_reserved_growth(n); } // Extension API: support for heterogeneous keys. @@ -1722,9 +2119,9 @@ class raw_hash_set { // Avoid probing if we won't be able to prefetch the addresses received. #ifdef Y_ABSL_INTERNAL_HAVE_PREFETCH prefetch_heap_block(); - auto seq = probe(ctrl_, hash_ref()(key), capacity_); - base_internal::PrefetchT0(ctrl_ + seq.offset()); - base_internal::PrefetchT0(slots_ + seq.offset()); + auto seq = probe(common(), hash_ref()(key)); + base_internal::PrefetchT0(control() + seq.offset()); + base_internal::PrefetchT0(slot_array() + seq.offset()); #endif // Y_ABSL_INTERNAL_HAVE_PREFETCH } @@ -1737,18 +2134,20 @@ class raw_hash_set { // called heterogeneous key support. template <class K = key_type> iterator find(const key_arg<K>& key, size_t hash) { - auto seq = probe(ctrl_, hash, capacity_); + auto seq = probe(common(), hash); + slot_type* slot_ptr = slot_array(); + const ctrl_t* ctrl = control(); while (true) { - Group g{ctrl_ + seq.offset()}; + Group g{ctrl + seq.offset()}; for (uint32_t i : g.Match(H2(hash))) { if (Y_ABSL_PREDICT_TRUE(PolicyTraits::apply( EqualElement<K>{key, eq_ref()}, - PolicyTraits::element(slots_ + seq.offset(i))))) + PolicyTraits::element(slot_ptr + seq.offset(i))))) return iterator_at(seq.offset(i)); } if (Y_ABSL_PREDICT_TRUE(g.MaskEmpty())) return end(); seq.next(); - assert(seq.index() <= capacity_ && "full table!"); + assert(seq.index() <= capacity() && "full table!"); } } template <class K = key_type> @@ -1786,9 +2185,9 @@ class raw_hash_set { return {it, it}; } - size_t bucket_count() const { return capacity_; } + size_t bucket_count() const { return capacity(); } float load_factor() const { - return capacity_ ? static_cast<double>(size()) / capacity_ : 0.0; + return capacity() ? static_cast<double>(size()) / capacity() : 0.0; } float max_load_factor() const { return 1.0f; } void max_load_factor(float) { @@ -1875,7 +2274,8 @@ class raw_hash_set { std::pair<iterator, bool> operator()(const K& key, Args&&...) && { auto res = s.find_or_prepare_insert(key); if (res.second) { - PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot); + PolicyTraits::transfer(&s.alloc_ref(), s.slot_array() + res.first, + &slot); } else if (do_destroy) { PolicyTraits::destroy(&s.alloc_ref(), &slot); } @@ -1891,102 +2291,43 @@ class raw_hash_set { // This merely updates the pertinent control byte. This can be used in // conjunction with Policy::transfer to move the object to another place. void erase_meta_only(const_iterator it) { - assert(IsFull(*it.inner_.ctrl_) && "erasing a dangling iterator"); - --size_; - const size_t index = static_cast<size_t>(it.inner_.ctrl_ - ctrl_); - const size_t index_before = (index - Group::kWidth) & capacity_; - const auto empty_after = Group(it.inner_.ctrl_).MaskEmpty(); - const auto empty_before = Group(ctrl_ + index_before).MaskEmpty(); - - // We count how many consecutive non empties we have to the right and to the - // left of `it`. If the sum is >= kWidth then there is at least one probe - // window that might have seen a full group. - bool was_never_full = - empty_before && empty_after && - static_cast<size_t>(empty_after.TrailingZeros() + - empty_before.LeadingZeros()) < Group::kWidth; - - SetCtrl(index, was_never_full ? ctrl_t::kEmpty : ctrl_t::kDeleted, - capacity_, ctrl_, slots_, sizeof(slot_type)); - growth_left() += was_never_full; - infoz().RecordErase(); + EraseMetaOnly(common(), it.inner_.ctrl_, sizeof(slot_type)); } // Allocates a backing array for `self` and initializes its control bytes. - // This reads `capacity_` and updates all other fields based on the result of + // This reads `capacity` and updates all other fields based on the result of // the allocation. // - // This does not free the currently held array; `capacity_` must be nonzero. - void initialize_slots() { - assert(capacity_); - // Folks with custom allocators often make unwarranted assumptions about the - // behavior of their classes vis-a-vis trivial destructability and what - // calls they will or wont make. Avoid sampling for people with custom - // allocators to get us out of this mess. This is not a hard guarantee but - // a workaround while we plan the exact guarantee we want to provide. - // + // This does not free the currently held array; `capacity` must be nonzero. + inline void initialize_slots() { // People are often sloppy with the exact type of their allocator (sometimes // it has an extra const or is missing the pair, but rebinds made it work - // anyway). To avoid the ambiguity, we work off SlotAlloc which we have - // bound more carefully. - if (std::is_same<SlotAlloc, std::allocator<slot_type>>::value && - slots_ == nullptr) { - infoz() = Sample(sizeof(slot_type)); - } - - char* mem = static_cast<char*>(Allocate<alignof(slot_type)>( - &alloc_ref(), - AllocSize(capacity_, sizeof(slot_type), alignof(slot_type)))); - ctrl_ = reinterpret_cast<ctrl_t*>(mem); - slots_ = reinterpret_cast<slot_type*>( - mem + SlotOffset(capacity_, alignof(slot_type))); - ResetCtrl(capacity_, ctrl_, slots_, sizeof(slot_type)); - reset_growth_left(); - infoz().RecordStorageChanged(size_, capacity_); - } - - // Destroys all slots in the backing array, frees the backing array, and - // clears all top-level book-keeping data. - // - // This essentially implements `map = raw_hash_set();`. - void destroy_slots() { - if (!capacity_) return; - for (size_t i = 0; i != capacity_; ++i) { - if (IsFull(ctrl_[i])) { - PolicyTraits::destroy(&alloc_ref(), slots_ + i); - } - } - - // Unpoison before returning the memory to the allocator. - SanitizerUnpoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_); - Deallocate<alignof(slot_type)>( - &alloc_ref(), ctrl_, - AllocSize(capacity_, sizeof(slot_type), alignof(slot_type))); - ctrl_ = EmptyGroup(); - slots_ = nullptr; - size_ = 0; - capacity_ = 0; - growth_left() = 0; + // anyway). + using CharAlloc = + typename y_absl::allocator_traits<Alloc>::template rebind_alloc<char>; + InitializeSlots<CharAlloc, sizeof(slot_type), alignof(slot_type)>( + common(), CharAlloc(alloc_ref())); } - void resize(size_t new_capacity) { + Y_ABSL_ATTRIBUTE_NOINLINE void resize(size_t new_capacity) { assert(IsValidCapacity(new_capacity)); - auto* old_ctrl = ctrl_; - auto* old_slots = slots_; - const size_t old_capacity = capacity_; - capacity_ = new_capacity; + auto* old_ctrl = control(); + auto* old_slots = slot_array(); + const size_t old_capacity = common().capacity_; + common().capacity_ = new_capacity; initialize_slots(); + auto* new_slots = slot_array(); size_t total_probe_length = 0; for (size_t i = 0; i != old_capacity; ++i) { if (IsFull(old_ctrl[i])) { size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, PolicyTraits::element(old_slots + i)); - auto target = find_first_non_full(ctrl_, hash, capacity_); + auto target = find_first_non_full(common(), hash); size_t new_i = target.offset; total_probe_length += target.probe_length; - SetCtrl(new_i, H2(hash), capacity_, ctrl_, slots_, sizeof(slot_type)); - PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, old_slots + i); + SetCtrl(common(), new_i, H2(hash), sizeof(slot_type)); + PolicyTraits::transfer(&alloc_ref(), new_slots + new_i, old_slots + i); } } if (old_capacity) { @@ -2002,70 +2343,10 @@ class raw_hash_set { // Prunes control bytes to remove as many tombstones as possible. // // See the comment on `rehash_and_grow_if_necessary()`. - void drop_deletes_without_resize() Y_ABSL_ATTRIBUTE_NOINLINE { - assert(IsValidCapacity(capacity_)); - assert(!is_small(capacity_)); - // Algorithm: - // - mark all DELETED slots as EMPTY - // - mark all FULL slots as DELETED - // - for each slot marked as DELETED - // hash = Hash(element) - // target = find_first_non_full(hash) - // if target is in the same group - // mark slot as FULL - // else if target is EMPTY - // transfer element to target - // mark slot as EMPTY - // mark target as FULL - // else if target is DELETED - // swap current element with target element - // mark target as FULL - // repeat procedure for current slot with moved from element (target) - ConvertDeletedToEmptyAndFullToDeleted(ctrl_, capacity_); - alignas(slot_type) unsigned char raw[sizeof(slot_type)]; - size_t total_probe_length = 0; - slot_type* slot = reinterpret_cast<slot_type*>(&raw); - for (size_t i = 0; i != capacity_; ++i) { - if (!IsDeleted(ctrl_[i])) continue; - const size_t hash = PolicyTraits::apply( - HashElement{hash_ref()}, PolicyTraits::element(slots_ + i)); - const FindInfo target = find_first_non_full(ctrl_, hash, capacity_); - const size_t new_i = target.offset; - total_probe_length += target.probe_length; - - // Verify if the old and new i fall within the same group wrt the hash. - // If they do, we don't need to move the object as it falls already in the - // best probe we can. - const size_t probe_offset = probe(ctrl_, hash, capacity_).offset(); - const auto probe_index = [probe_offset, this](size_t pos) { - return ((pos - probe_offset) & capacity_) / Group::kWidth; - }; - - // Element doesn't move. - if (Y_ABSL_PREDICT_TRUE(probe_index(new_i) == probe_index(i))) { - SetCtrl(i, H2(hash), capacity_, ctrl_, slots_, sizeof(slot_type)); - continue; - } - if (IsEmpty(ctrl_[new_i])) { - // Transfer element to the empty spot. - // SetCtrl poisons/unpoisons the slots so we have to call it at the - // right time. - SetCtrl(new_i, H2(hash), capacity_, ctrl_, slots_, sizeof(slot_type)); - PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slots_ + i); - SetCtrl(i, ctrl_t::kEmpty, capacity_, ctrl_, slots_, sizeof(slot_type)); - } else { - assert(IsDeleted(ctrl_[new_i])); - SetCtrl(new_i, H2(hash), capacity_, ctrl_, slots_, sizeof(slot_type)); - // Until we are done rehashing, DELETED marks previously FULL slots. - // Swap i and new_i elements. - PolicyTraits::transfer(&alloc_ref(), slot, slots_ + i); - PolicyTraits::transfer(&alloc_ref(), slots_ + i, slots_ + new_i); - PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slot); - --i; // repeat - } - } - reset_growth_left(); - infoz().RecordRehash(total_probe_length); + inline void drop_deletes_without_resize() { + // Stack-allocate space for swapping elements. + alignas(slot_type) unsigned char tmp[sizeof(slot_type)]; + DropDeletesWithoutResize(common(), GetPolicyFunctions(), tmp); } // Called whenever the table *might* need to conditionally grow. @@ -2074,14 +2355,13 @@ class raw_hash_set { // growth is unnecessary, because vacating tombstones is beneficial for // performance in the long-run. void rehash_and_grow_if_necessary() { - if (capacity_ == 0) { - resize(1); - } else if (capacity_ > Group::kWidth && - // Do these calcuations in 64-bit to avoid overflow. - size() * uint64_t{32} <= capacity_ * uint64_t{25}) { + const size_t cap = capacity(); + if (cap > Group::kWidth && + // Do these calcuations in 64-bit to avoid overflow. + size() * uint64_t{32} <= cap* uint64_t{25}) { // Squash DELETED without growing if there is enough capacity. // - // Rehash in place if the current size is <= 25/32 of capacity_. + // Rehash in place if the current size is <= 25/32 of capacity. // Rationale for such a high factor: 1) drop_deletes_without_resize() is // faster than resize, and 2) it takes quite a bit of work to add // tombstones. In the worst case, seems to take approximately 4 @@ -2099,8 +2379,8 @@ class raw_hash_set { // // Here is output of an experiment using the BM_CacheInSteadyState // benchmark running the old case (where we rehash-in-place only if we can - // reclaim at least 7/16*capacity_) vs. this code (which rehashes in place - // if we can recover 3/32*capacity_). + // reclaim at least 7/16*capacity) vs. this code (which rehashes in place + // if we can recover 3/32*capacity). // // Note that although in the worst-case number of rehashes jumped up from // 15 to 190, but the number of operations per second is almost the same. @@ -2123,23 +2403,24 @@ class raw_hash_set { drop_deletes_without_resize(); } else { // Otherwise grow the container. - resize(capacity_ * 2 + 1); + resize(NextCapacity(cap)); } } bool has_element(const value_type& elem) const { size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, elem); - auto seq = probe(ctrl_, hash, capacity_); + auto seq = probe(common(), hash); + const ctrl_t* ctrl = control(); while (true) { - Group g{ctrl_ + seq.offset()}; + Group g{ctrl + seq.offset()}; for (uint32_t i : g.Match(H2(hash))) { - if (Y_ABSL_PREDICT_TRUE(PolicyTraits::element(slots_ + seq.offset(i)) == - elem)) + if (Y_ABSL_PREDICT_TRUE( + PolicyTraits::element(slot_array() + seq.offset(i)) == elem)) return true; } if (Y_ABSL_PREDICT_TRUE(g.MaskEmpty())) return false; seq.next(); - assert(seq.index() <= capacity_ && "full table!"); + assert(seq.index() <= capacity() && "full table!"); } return false; } @@ -2164,18 +2445,19 @@ class raw_hash_set { std::pair<size_t, bool> find_or_prepare_insert(const K& key) { prefetch_heap_block(); auto hash = hash_ref()(key); - auto seq = probe(ctrl_, hash, capacity_); + auto seq = probe(common(), hash); + const ctrl_t* ctrl = control(); while (true) { - Group g{ctrl_ + seq.offset()}; + Group g{ctrl + seq.offset()}; for (uint32_t i : g.Match(H2(hash))) { if (Y_ABSL_PREDICT_TRUE(PolicyTraits::apply( EqualElement<K>{key, eq_ref()}, - PolicyTraits::element(slots_ + seq.offset(i))))) + PolicyTraits::element(slot_array() + seq.offset(i))))) return {seq.offset(i), false}; } if (Y_ABSL_PREDICT_TRUE(g.MaskEmpty())) break; seq.next(); - assert(seq.index() <= capacity_ && "full table!"); + assert(seq.index() <= capacity() && "full table!"); } return {prepare_insert(hash), true}; } @@ -2185,16 +2467,24 @@ class raw_hash_set { // // REQUIRES: At least one non-full slot available. size_t prepare_insert(size_t hash) Y_ABSL_ATTRIBUTE_NOINLINE { - auto target = find_first_non_full(ctrl_, hash, capacity_); - if (Y_ABSL_PREDICT_FALSE(growth_left() == 0 && - !IsDeleted(ctrl_[target.offset]))) { + const bool rehash_for_bug_detection = + common().should_rehash_for_bug_detection_on_insert(); + if (rehash_for_bug_detection) { + // Move to a different heap allocation in order to detect bugs. + const size_t cap = capacity(); + resize(growth_left() > 0 ? cap : NextCapacity(cap)); + } + auto target = find_first_non_full(common(), hash); + if (!rehash_for_bug_detection && + Y_ABSL_PREDICT_FALSE(growth_left() == 0 && + !IsDeleted(control()[target.offset]))) { rehash_and_grow_if_necessary(); - target = find_first_non_full(ctrl_, hash, capacity_); + target = find_first_non_full(common(), hash); } - ++size_; - growth_left() -= IsEmpty(ctrl_[target.offset]); - SetCtrl(target.offset, H2(hash), capacity_, ctrl_, slots_, - sizeof(slot_type)); + ++common().size_; + growth_left() -= IsEmpty(control()[target.offset]); + SetCtrl(common(), target.offset, H2(hash), sizeof(slot_type)); + common().maybe_increment_generation_on_insert(); infoz().RecordInsert(hash, target.probe_length); return target.offset; } @@ -2209,7 +2499,7 @@ class raw_hash_set { // POSTCONDITION: *m.iterator_at(i) == value_type(forward<Args>(args)...). template <class... Args> void emplace_at(size_t i, Args&&... args) { - PolicyTraits::construct(&alloc_ref(), slots_ + i, + PolicyTraits::construct(&alloc_ref(), slot_array() + i, std::forward<Args>(args)...); assert(PolicyTraits::apply(FindElement{*this}, *iterator_at(i)) == @@ -2217,16 +2507,16 @@ class raw_hash_set { "constructed value does not match the lookup key"); } - iterator iterator_at(size_t i) { return {ctrl_ + i, slots_ + i}; } - const_iterator iterator_at(size_t i) const { return {ctrl_ + i, slots_ + i}; } + iterator iterator_at(size_t i) { + return {control() + i, slot_array() + i, common().generation_ptr()}; + } + const_iterator iterator_at(size_t i) const { + return {control() + i, slot_array() + i, common().generation_ptr()}; + } private: friend struct RawHashSetTestOnlyAccess; - void reset_growth_left() { - growth_left() = CapacityToGrowth(capacity()) - size_; - } - // The number of slots we can still fill without needing to rehash. // // This is stored separately due to tombstones: we do not include tombstones @@ -2237,49 +2527,76 @@ class raw_hash_set { // side-effect. // // See `CapacityToGrowth()`. - size_t& growth_left() { return settings_.template get<0>(); } + size_t& growth_left() { return common().growth_left(); } // Prefetch the heap-allocated memory region to resolve potential TLB misses. // This is intended to overlap with execution of calculating the hash for a // key. - void prefetch_heap_block() const { - base_internal::PrefetchT2(ctrl_); - } + void prefetch_heap_block() const { base_internal::PrefetchT2(control()); } + + CommonFields& common() { return settings_.template get<0>(); } + const CommonFields& common() const { return settings_.template get<0>(); } - HashtablezInfoHandle& infoz() { return settings_.template get<1>(); } + ctrl_t* control() const { return common().control_; } + slot_type* slot_array() const { + return static_cast<slot_type*>(common().slots_); + } + HashtablezInfoHandle& infoz() { return common().infoz(); } - hasher& hash_ref() { return settings_.template get<2>(); } - const hasher& hash_ref() const { return settings_.template get<2>(); } - key_equal& eq_ref() { return settings_.template get<3>(); } - const key_equal& eq_ref() const { return settings_.template get<3>(); } - allocator_type& alloc_ref() { return settings_.template get<4>(); } + hasher& hash_ref() { return settings_.template get<1>(); } + const hasher& hash_ref() const { return settings_.template get<1>(); } + key_equal& eq_ref() { return settings_.template get<2>(); } + const key_equal& eq_ref() const { return settings_.template get<2>(); } + allocator_type& alloc_ref() { return settings_.template get<3>(); } const allocator_type& alloc_ref() const { - return settings_.template get<4>(); + return settings_.template get<3>(); } - // TODO(alkis): Investigate removing some of these fields: - // - ctrl/slots can be derived from each other - // - size can be moved into the slot array + // Make type-specific functions for this type's PolicyFunctions struct. + static size_t hash_slot_fn(void* set, void* slot) { + auto* h = static_cast<raw_hash_set*>(set); + return PolicyTraits::apply( + HashElement{h->hash_ref()}, + PolicyTraits::element(static_cast<slot_type*>(slot))); + } + static void transfer_slot_fn(void* set, void* dst, void* src) { + auto* h = static_cast<raw_hash_set*>(set); + PolicyTraits::transfer(&h->alloc_ref(), static_cast<slot_type*>(dst), + static_cast<slot_type*>(src)); + } + // Note: dealloc_fn will only be used if we have a non-standard allocator. + static void dealloc_fn(void* set, const PolicyFunctions&, ctrl_t* ctrl, + void* slot_mem, size_t n) { + auto* h = static_cast<raw_hash_set*>(set); - // The control bytes (and, also, a pointer to the base of the backing array). - // - // This contains `capacity_ + 1 + NumClonedBytes()` entries, even - // when the table is empty (hence EmptyGroup). - ctrl_t* ctrl_ = EmptyGroup(); - // The beginning of the slots, located at `SlotOffset()` bytes after - // `ctrl_`. May be null for empty tables. - slot_type* slots_ = nullptr; + // Unpoison before returning the memory to the allocator. + SanitizerUnpoisonMemoryRegion(slot_mem, sizeof(slot_type) * n); - // The number of filled slots. - size_t size_ = 0; + Deallocate<alignof(slot_type)>( + &h->alloc_ref(), ctrl, + AllocSize(n, sizeof(slot_type), alignof(slot_type))); + } + + static const PolicyFunctions& GetPolicyFunctions() { + static constexpr PolicyFunctions value = { + sizeof(slot_type), + &raw_hash_set::hash_slot_fn, + PolicyTraits::transfer_uses_memcpy() + ? TransferRelocatable<sizeof(slot_type)> + : &raw_hash_set::transfer_slot_fn, + (std::is_same<SlotAlloc, std::allocator<slot_type>>::value + ? &DeallocateStandard<alignof(slot_type)> + : &raw_hash_set::dealloc_fn), + }; + return value; + } - // The total number of available slots. - size_t capacity_ = 0; - y_absl::container_internal::CompressedTuple<size_t /* growth_left */, - HashtablezInfoHandle, hasher, - key_equal, allocator_type> - settings_{0u, HashtablezInfoHandle{}, hasher{}, key_equal{}, - allocator_type{}}; + // Bundle together CommonFields plus other objects which might be empty. + // CompressedTuple will ensure that sizeof is not affected by any of the empty + // fields that occur after CommonFields. + y_absl::container_internal::CompressedTuple<CommonFields, hasher, key_equal, + allocator_type> + settings_{CommonFields{}, hasher{}, key_equal{}, allocator_type{}}; }; // Erases all elements that satisfy the predicate `pred` from the container `c`. @@ -2307,14 +2624,15 @@ struct HashtableDebugAccess<Set, y_absl::void_t<typename Set::raw_hash_set>> { const typename Set::key_type& key) { size_t num_probes = 0; size_t hash = set.hash_ref()(key); - auto seq = probe(set.ctrl_, hash, set.capacity_); + auto seq = probe(set.common(), hash); + const ctrl_t* ctrl = set.control(); while (true) { - container_internal::Group g{set.ctrl_ + seq.offset()}; + container_internal::Group g{ctrl + seq.offset()}; for (uint32_t i : g.Match(container_internal::H2(hash))) { if (Traits::apply( typename Set::template EqualElement<typename Set::key_type>{ key, set.eq_ref()}, - Traits::element(set.slots_ + seq.offset(i)))) + Traits::element(set.slot_array() + seq.offset(i)))) return num_probes; ++num_probes; } @@ -2325,7 +2643,7 @@ struct HashtableDebugAccess<Set, y_absl::void_t<typename Set::raw_hash_set>> { } static size_t AllocatedByteSize(const Set& c) { - size_t capacity = c.capacity_; + size_t capacity = c.capacity(); if (capacity == 0) return 0; size_t m = AllocSize(capacity, sizeof(Slot), alignof(Slot)); @@ -2333,9 +2651,10 @@ struct HashtableDebugAccess<Set, y_absl::void_t<typename Set::raw_hash_set>> { if (per_slot != ~size_t{}) { m += per_slot * c.size(); } else { + const ctrl_t* ctrl = c.control(); for (size_t i = 0; i != capacity; ++i) { - if (container_internal::IsFull(c.ctrl_[i])) { - m += Traits::space_used(c.slots_ + i); + if (container_internal::IsFull(ctrl[i])) { + m += Traits::space_used(c.slot_array() + i); } } } @@ -2360,6 +2679,7 @@ struct HashtableDebugAccess<Set, y_absl::void_t<typename Set::raw_hash_set>> { Y_ABSL_NAMESPACE_END } // namespace y_absl +#undef Y_ABSL_SWISSTABLE_ENABLE_GENERATIONS #undef Y_ABSL_INTERNAL_ASSERT_IS_FULL #endif // Y_ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ |