diff options
author | anastasy888 <anastasy888@yandex-team.ru> | 2022-02-10 16:45:54 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:54 +0300 |
commit | 49f765d71da452ea93138a25559dfa68dd76c7f3 (patch) | |
tree | 1016041feb637349e401dcc0fa85217dd2c2c639 /contrib/restricted/abseil-cpp/absl/container/internal/raw_hash_set.h | |
parent | 7353a3fdea9c67c256980c00a2b3b67f09b23a27 (diff) | |
download | ydb-49f765d71da452ea93138a25559dfa68dd76c7f3.tar.gz |
Restoring authorship annotation for <anastasy888@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'contrib/restricted/abseil-cpp/absl/container/internal/raw_hash_set.h')
-rw-r--r-- | contrib/restricted/abseil-cpp/absl/container/internal/raw_hash_set.h | 3180 |
1 files changed, 1590 insertions, 1590 deletions
diff --git a/contrib/restricted/abseil-cpp/absl/container/internal/raw_hash_set.h b/contrib/restricted/abseil-cpp/absl/container/internal/raw_hash_set.h index 12682b3532..1364f8cca7 100644 --- a/contrib/restricted/abseil-cpp/absl/container/internal/raw_hash_set.h +++ b/contrib/restricted/abseil-cpp/absl/container/internal/raw_hash_set.h @@ -1,92 +1,92 @@ -// Copyright 2018 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// An open-addressing -// hashtable with quadratic probing. -// -// This is a low level hashtable on top of which different interfaces can be -// implemented, like flat_hash_set, node_hash_set, string_hash_set, etc. -// -// The table interface is similar to that of std::unordered_set. Notable -// differences are that most member functions support heterogeneous keys when -// BOTH the hash and eq functions are marked as transparent. They do so by -// providing a typedef called `is_transparent`. -// -// When heterogeneous lookup is enabled, functions that take key_type act as if -// they have an overload set like: -// -// iterator find(const key_type& key); -// template <class K> -// iterator find(const K& key); -// -// size_type erase(const key_type& key); -// template <class K> -// size_type erase(const K& key); -// -// std::pair<iterator, iterator> equal_range(const key_type& key); -// template <class K> -// std::pair<iterator, iterator> equal_range(const K& key); -// -// When heterogeneous lookup is disabled, only the explicit `key_type` overloads -// exist. -// -// find() also supports passing the hash explicitly: -// -// iterator find(const key_type& key, size_t hash); -// template <class U> -// iterator find(const U& key, size_t hash); -// -// In addition the pointer to element and iterator stability guarantees are -// weaker: all iterators and pointers are invalidated after a new element is -// inserted. -// -// IMPLEMENTATION DETAILS -// -// The table stores elements inline in a slot array. In addition to the slot -// array the table maintains some control state per slot. The extra state is one -// byte per slot and stores empty or deleted marks, or alternatively 7 bits from -// the hash of an occupied slot. The table is split into logical groups of -// slots, like so: -// -// Group 1 Group 2 Group 3 -// +---------------+---------------+---------------+ -// | | | | | | | | | | | | | | | | | | | | | | | | | -// +---------------+---------------+---------------+ -// -// On lookup the hash is split into two parts: -// - H2: 7 bits (those stored in the control bytes) -// - H1: the rest of the bits -// The groups are probed using H1. For each group the slots are matched to H2 in -// parallel. Because H2 is 7 bits (128 states) and the number of slots per group -// is low (8 or 16) in almost all cases a match in H2 is also a lookup hit. -// -// On insert, once the right group is found (as in lookup), its slots are -// filled in order. -// -// On erase a slot is cleared. In case the group did not have any empty slots -// before the erase, the erased slot is marked as deleted. -// -// Groups without empty slots (but maybe with deleted slots) extend the probe -// sequence. The probing algorithm is quadratic. Given N the number of groups, -// the probing function for the i'th probe is: -// -// P(0) = H1 % N -// -// P(i) = (P(i - 1) + i) % N -// -// This probing function guarantees that after N probes, all the groups of the -// table will be probed exactly once. +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// An open-addressing +// hashtable with quadratic probing. +// +// This is a low level hashtable on top of which different interfaces can be +// implemented, like flat_hash_set, node_hash_set, string_hash_set, etc. +// +// The table interface is similar to that of std::unordered_set. Notable +// differences are that most member functions support heterogeneous keys when +// BOTH the hash and eq functions are marked as transparent. They do so by +// providing a typedef called `is_transparent`. +// +// When heterogeneous lookup is enabled, functions that take key_type act as if +// they have an overload set like: +// +// iterator find(const key_type& key); +// template <class K> +// iterator find(const K& key); +// +// size_type erase(const key_type& key); +// template <class K> +// size_type erase(const K& key); +// +// std::pair<iterator, iterator> equal_range(const key_type& key); +// template <class K> +// std::pair<iterator, iterator> equal_range(const K& key); +// +// When heterogeneous lookup is disabled, only the explicit `key_type` overloads +// exist. +// +// find() also supports passing the hash explicitly: +// +// iterator find(const key_type& key, size_t hash); +// template <class U> +// iterator find(const U& key, size_t hash); +// +// In addition the pointer to element and iterator stability guarantees are +// weaker: all iterators and pointers are invalidated after a new element is +// inserted. +// +// IMPLEMENTATION DETAILS +// +// The table stores elements inline in a slot array. In addition to the slot +// array the table maintains some control state per slot. The extra state is one +// byte per slot and stores empty or deleted marks, or alternatively 7 bits from +// the hash of an occupied slot. The table is split into logical groups of +// slots, like so: +// +// Group 1 Group 2 Group 3 +// +---------------+---------------+---------------+ +// | | | | | | | | | | | | | | | | | | | | | | | | | +// +---------------+---------------+---------------+ +// +// On lookup the hash is split into two parts: +// - H2: 7 bits (those stored in the control bytes) +// - H1: the rest of the bits +// The groups are probed using H1. For each group the slots are matched to H2 in +// parallel. Because H2 is 7 bits (128 states) and the number of slots per group +// is low (8 or 16) in almost all cases a match in H2 is also a lookup hit. +// +// On insert, once the right group is found (as in lookup), its slots are +// filled in order. +// +// On erase a slot is cleared. In case the group did not have any empty slots +// before the erase, the erased slot is marked as deleted. +// +// Groups without empty slots (but maybe with deleted slots) extend the probe +// sequence. The probing algorithm is quadratic. Given N the number of groups, +// the probing function for the i'th probe is: +// +// P(0) = H1 % N +// +// P(i) = (P(i - 1) + i) % N +// +// This probing function guarantees that after N probes, all the groups of the +// table will be probed exactly once. // // The control state and slot array are stored contiguously in a shared heap // allocation. The layout of this allocation is: `capacity()` control bytes, @@ -98,40 +98,40 @@ // which there are more than `capacity()` cloned control bytes, the extra bytes // are `kEmpty`, and these ensure that we always see at least one empty slot and // can stop an unsuccessful search. - -#ifndef ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ -#define ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ - -#include <algorithm> -#include <cmath> -#include <cstdint> -#include <cstring> -#include <iterator> -#include <limits> -#include <memory> -#include <tuple> -#include <type_traits> -#include <utility> - -#include "absl/base/internal/endian.h" + +#ifndef ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ +#define ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ + +#include <algorithm> +#include <cmath> +#include <cstdint> +#include <cstring> +#include <iterator> +#include <limits> +#include <memory> +#include <tuple> +#include <type_traits> +#include <utility> + +#include "absl/base/internal/endian.h" #include "absl/base/optimization.h" -#include "absl/base/port.h" -#include "absl/container/internal/common.h" -#include "absl/container/internal/compressed_tuple.h" -#include "absl/container/internal/container_memory.h" -#include "absl/container/internal/hash_policy_traits.h" -#include "absl/container/internal/hashtable_debug_hooks.h" -#include "absl/container/internal/hashtablez_sampler.h" -#include "absl/container/internal/have_sse.h" -#include "absl/memory/memory.h" -#include "absl/meta/type_traits.h" +#include "absl/base/port.h" +#include "absl/container/internal/common.h" +#include "absl/container/internal/compressed_tuple.h" +#include "absl/container/internal/container_memory.h" +#include "absl/container/internal/hash_policy_traits.h" +#include "absl/container/internal/hashtable_debug_hooks.h" +#include "absl/container/internal/hashtablez_sampler.h" +#include "absl/container/internal/have_sse.h" +#include "absl/memory/memory.h" +#include "absl/meta/type_traits.h" #include "absl/numeric/bits.h" -#include "absl/utility/utility.h" - -namespace absl { +#include "absl/utility/utility.h" + +namespace absl { ABSL_NAMESPACE_BEGIN -namespace container_internal { - +namespace container_internal { + template <typename AllocType> void SwapAlloc(AllocType& lhs, AllocType& rhs, std::true_type /* propagate_on_container_swap */) { @@ -142,141 +142,141 @@ template <typename AllocType> void SwapAlloc(AllocType& /*lhs*/, AllocType& /*rhs*/, std::false_type /* propagate_on_container_swap */) {} -template <size_t Width> -class probe_seq { - public: - probe_seq(size_t hash, size_t mask) { - assert(((mask + 1) & mask) == 0 && "not a mask"); - mask_ = mask; - offset_ = hash & mask_; - } - size_t offset() const { return offset_; } - size_t offset(size_t i) const { return (offset_ + i) & mask_; } - - void next() { - index_ += Width; - offset_ += index_; - offset_ &= mask_; - } - // 0-based probe index. The i-th probe in the probe sequence. - size_t index() const { return index_; } - - private: - size_t mask_; - size_t offset_; - size_t index_ = 0; -}; - -template <class ContainerKey, class Hash, class Eq> -struct RequireUsableKey { - template <class PassedKey, class... Args> - std::pair< - decltype(std::declval<const Hash&>()(std::declval<const PassedKey&>())), - decltype(std::declval<const Eq&>()(std::declval<const ContainerKey&>(), - std::declval<const PassedKey&>()))>* - operator()(const PassedKey&, const Args&...) const; -}; - -template <class E, class Policy, class Hash, class Eq, class... Ts> -struct IsDecomposable : std::false_type {}; - -template <class Policy, class Hash, class Eq, class... Ts> -struct IsDecomposable< - absl::void_t<decltype( - Policy::apply(RequireUsableKey<typename Policy::key_type, Hash, Eq>(), - std::declval<Ts>()...))>, - Policy, Hash, Eq, Ts...> : std::true_type {}; - -// TODO(alkis): Switch to std::is_nothrow_swappable when gcc/clang supports it. -template <class T> +template <size_t Width> +class probe_seq { + public: + probe_seq(size_t hash, size_t mask) { + assert(((mask + 1) & mask) == 0 && "not a mask"); + mask_ = mask; + offset_ = hash & mask_; + } + size_t offset() const { return offset_; } + size_t offset(size_t i) const { return (offset_ + i) & mask_; } + + void next() { + index_ += Width; + offset_ += index_; + offset_ &= mask_; + } + // 0-based probe index. The i-th probe in the probe sequence. + size_t index() const { return index_; } + + private: + size_t mask_; + size_t offset_; + size_t index_ = 0; +}; + +template <class ContainerKey, class Hash, class Eq> +struct RequireUsableKey { + template <class PassedKey, class... Args> + std::pair< + decltype(std::declval<const Hash&>()(std::declval<const PassedKey&>())), + decltype(std::declval<const Eq&>()(std::declval<const ContainerKey&>(), + std::declval<const PassedKey&>()))>* + operator()(const PassedKey&, const Args&...) const; +}; + +template <class E, class Policy, class Hash, class Eq, class... Ts> +struct IsDecomposable : std::false_type {}; + +template <class Policy, class Hash, class Eq, class... Ts> +struct IsDecomposable< + absl::void_t<decltype( + Policy::apply(RequireUsableKey<typename Policy::key_type, Hash, Eq>(), + std::declval<Ts>()...))>, + Policy, Hash, Eq, Ts...> : std::true_type {}; + +// TODO(alkis): Switch to std::is_nothrow_swappable when gcc/clang supports it. +template <class T> constexpr bool IsNoThrowSwappable(std::true_type = {} /* is_swappable */) { - using std::swap; - return noexcept(swap(std::declval<T&>(), std::declval<T&>())); -} + using std::swap; + return noexcept(swap(std::declval<T&>(), std::declval<T&>())); +} template <class T> constexpr bool IsNoThrowSwappable(std::false_type /* is_swappable */) { return false; } - -template <typename T> + +template <typename T> uint32_t TrailingZeros(T x) { ABSL_INTERNAL_ASSUME(x != 0); return countr_zero(x); -} - -// An abstraction over a bitmask. It provides an easy way to iterate through the -// indexes of the set bits of a bitmask. When Shift=0 (platforms with SSE), -// this is a true bitmask. On non-SSE, platforms the arithematic used to -// emulate the SSE behavior works in bytes (Shift=3) and leaves each bytes as -// either 0x00 or 0x80. -// -// For example: -// for (int i : BitMask<uint32_t, 16>(0x5)) -> yields 0, 2 -// for (int i : BitMask<uint64_t, 8, 3>(0x0000000080800000)) -> yields 2, 3 -template <class T, int SignificantBits, int Shift = 0> -class BitMask { - static_assert(std::is_unsigned<T>::value, ""); - static_assert(Shift == 0 || Shift == 3, ""); - - public: - // These are useful for unit tests (gunit). - using value_type = int; - using iterator = BitMask; - using const_iterator = BitMask; - - explicit BitMask(T mask) : mask_(mask) {} - BitMask& operator++() { - mask_ &= (mask_ - 1); - return *this; - } - explicit operator bool() const { return mask_ != 0; } - int operator*() const { return LowestBitSet(); } +} + +// An abstraction over a bitmask. It provides an easy way to iterate through the +// indexes of the set bits of a bitmask. When Shift=0 (platforms with SSE), +// this is a true bitmask. On non-SSE, platforms the arithematic used to +// emulate the SSE behavior works in bytes (Shift=3) and leaves each bytes as +// either 0x00 or 0x80. +// +// For example: +// for (int i : BitMask<uint32_t, 16>(0x5)) -> yields 0, 2 +// for (int i : BitMask<uint64_t, 8, 3>(0x0000000080800000)) -> yields 2, 3 +template <class T, int SignificantBits, int Shift = 0> +class BitMask { + static_assert(std::is_unsigned<T>::value, ""); + static_assert(Shift == 0 || Shift == 3, ""); + + public: + // These are useful for unit tests (gunit). + using value_type = int; + using iterator = BitMask; + using const_iterator = BitMask; + + explicit BitMask(T mask) : mask_(mask) {} + BitMask& operator++() { + mask_ &= (mask_ - 1); + return *this; + } + explicit operator bool() const { return mask_ != 0; } + int operator*() const { return LowestBitSet(); } uint32_t LowestBitSet() const { - return container_internal::TrailingZeros(mask_) >> Shift; - } + return container_internal::TrailingZeros(mask_) >> Shift; + } uint32_t HighestBitSet() const { return static_cast<uint32_t>((bit_width(mask_) - 1) >> Shift); - } - - BitMask begin() const { return *this; } - BitMask end() const { return BitMask(0); } - + } + + BitMask begin() const { return *this; } + BitMask end() const { return BitMask(0); } + uint32_t TrailingZeros() const { - return container_internal::TrailingZeros(mask_) >> Shift; - } - + return container_internal::TrailingZeros(mask_) >> Shift; + } + uint32_t LeadingZeros() const { - constexpr int total_significant_bits = SignificantBits << Shift; - constexpr int extra_bits = sizeof(T) * 8 - total_significant_bits; + constexpr int total_significant_bits = SignificantBits << Shift; + constexpr int extra_bits = sizeof(T) * 8 - total_significant_bits; return countl_zero(mask_ << extra_bits) >> Shift; - } - - private: - friend bool operator==(const BitMask& a, const BitMask& b) { - return a.mask_ == b.mask_; - } - friend bool operator!=(const BitMask& a, const BitMask& b) { - return a.mask_ != b.mask_; - } - - T mask_; -}; - -using h2_t = uint8_t; - -// The values here are selected for maximum performance. See the static asserts + } + + private: + friend bool operator==(const BitMask& a, const BitMask& b) { + return a.mask_ == b.mask_; + } + friend bool operator!=(const BitMask& a, const BitMask& b) { + return a.mask_ != b.mask_; + } + + T mask_; +}; + +using h2_t = uint8_t; + +// The values here are selected for maximum performance. See the static asserts // below for details. We use an enum class so that when strict aliasing is // enabled, the compiler knows ctrl_t doesn't alias other types. enum class ctrl_t : int8_t { - kEmpty = -128, // 0b10000000 - kDeleted = -2, // 0b11111110 - kSentinel = -1, // 0b11111111 -}; -static_assert( + kEmpty = -128, // 0b10000000 + kDeleted = -2, // 0b11111110 + kSentinel = -1, // 0b11111111 +}; +static_assert( (static_cast<int8_t>(ctrl_t::kEmpty) & static_cast<int8_t>(ctrl_t::kDeleted) & static_cast<int8_t>(ctrl_t::kSentinel) & 0x80) != 0, - "Special markers need to have the MSB to make checking for them efficient"); + "Special markers need to have the MSB to make checking for them efficient"); static_assert( ctrl_t::kEmpty < ctrl_t::kSentinel && ctrl_t::kDeleted < ctrl_t::kSentinel, "ctrl_t::kEmpty and ctrl_t::kDeleted must be smaller than " @@ -287,7 +287,7 @@ static_assert( "registers (pcmpeqd xmm, xmm)"); static_assert(ctrl_t::kEmpty == static_cast<ctrl_t>(-128), "ctrl_t::kEmpty must be -128 to make the SIMD check for its " - "existence efficient (psignb xmm, xmm)"); + "existence efficient (psignb xmm, xmm)"); static_assert( (~static_cast<int8_t>(ctrl_t::kEmpty) & ~static_cast<int8_t>(ctrl_t::kDeleted) & @@ -297,196 +297,196 @@ static_assert( "MatchEmptyOrDeleted() efficient"); static_assert(ctrl_t::kDeleted == static_cast<ctrl_t>(-2), "ctrl_t::kDeleted must be -2 to make the implementation of " - "ConvertSpecialToEmptyAndFullToDeleted efficient"); - -// A single block of empty control bytes for tables without any slots allocated. -// This enables removing a branch in the hot path of find(). + "ConvertSpecialToEmptyAndFullToDeleted efficient"); + +// A single block of empty control bytes for tables without any slots allocated. +// This enables removing a branch in the hot path of find(). ABSL_DLL extern const ctrl_t kEmptyGroup[16]; -inline ctrl_t* EmptyGroup() { +inline ctrl_t* EmptyGroup() { return const_cast<ctrl_t*>(kEmptyGroup); -} - -// Mixes a randomly generated per-process seed with `hash` and `ctrl` to -// randomize insertion order within groups. +} + +// Mixes a randomly generated per-process seed with `hash` and `ctrl` to +// randomize insertion order within groups. bool ShouldInsertBackwards(size_t hash, const ctrl_t* ctrl); - -// Returns a hash seed. -// -// The seed consists of the ctrl_ pointer, which adds enough entropy to ensure -// non-determinism of iteration order in most cases. -inline size_t HashSeed(const ctrl_t* ctrl) { - // The low bits of the pointer have little or no entropy because of - // alignment. We shift the pointer to try to use higher entropy bits. A - // good number seems to be 12 bits, because that aligns with page size. - return reinterpret_cast<uintptr_t>(ctrl) >> 12; -} - -inline size_t H1(size_t hash, const ctrl_t* ctrl) { - return (hash >> 7) ^ HashSeed(ctrl); -} + +// Returns a hash seed. +// +// The seed consists of the ctrl_ pointer, which adds enough entropy to ensure +// non-determinism of iteration order in most cases. +inline size_t HashSeed(const ctrl_t* ctrl) { + // The low bits of the pointer have little or no entropy because of + // alignment. We shift the pointer to try to use higher entropy bits. A + // good number seems to be 12 bits, because that aligns with page size. + return reinterpret_cast<uintptr_t>(ctrl) >> 12; +} + +inline size_t H1(size_t hash, const ctrl_t* ctrl) { + return (hash >> 7) ^ HashSeed(ctrl); +} inline h2_t H2(size_t hash) { return hash & 0x7F; } - + inline bool IsEmpty(ctrl_t c) { return c == ctrl_t::kEmpty; } inline bool IsFull(ctrl_t c) { return c >= static_cast<ctrl_t>(0); } inline bool IsDeleted(ctrl_t c) { return c == ctrl_t::kDeleted; } inline bool IsEmptyOrDeleted(ctrl_t c) { return c < ctrl_t::kSentinel; } - + #if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 - -// https://github.com/abseil/abseil-cpp/issues/209 -// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87853 -// _mm_cmpgt_epi8 is broken under GCC with -funsigned-char -// Work around this by using the portable implementation of Group -// when using -funsigned-char under GCC. -inline __m128i _mm_cmpgt_epi8_fixed(__m128i a, __m128i b) { -#if defined(__GNUC__) && !defined(__clang__) - if (std::is_unsigned<char>::value) { - const __m128i mask = _mm_set1_epi8(0x80); - const __m128i diff = _mm_subs_epi8(b, a); - return _mm_cmpeq_epi8(_mm_and_si128(diff, mask), mask); - } -#endif - return _mm_cmpgt_epi8(a, b); -} - -struct GroupSse2Impl { - static constexpr size_t kWidth = 16; // the number of slots per group - - explicit GroupSse2Impl(const ctrl_t* pos) { - ctrl = _mm_loadu_si128(reinterpret_cast<const __m128i*>(pos)); - } - - // Returns a bitmask representing the positions of slots that match hash. - BitMask<uint32_t, kWidth> Match(h2_t hash) const { - auto match = _mm_set1_epi8(hash); - return BitMask<uint32_t, kWidth>( - _mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl))); - } - - // Returns a bitmask representing the positions of empty slots. - BitMask<uint32_t, kWidth> MatchEmpty() const { + +// https://github.com/abseil/abseil-cpp/issues/209 +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87853 +// _mm_cmpgt_epi8 is broken under GCC with -funsigned-char +// Work around this by using the portable implementation of Group +// when using -funsigned-char under GCC. +inline __m128i _mm_cmpgt_epi8_fixed(__m128i a, __m128i b) { +#if defined(__GNUC__) && !defined(__clang__) + if (std::is_unsigned<char>::value) { + const __m128i mask = _mm_set1_epi8(0x80); + const __m128i diff = _mm_subs_epi8(b, a); + return _mm_cmpeq_epi8(_mm_and_si128(diff, mask), mask); + } +#endif + return _mm_cmpgt_epi8(a, b); +} + +struct GroupSse2Impl { + static constexpr size_t kWidth = 16; // the number of slots per group + + explicit GroupSse2Impl(const ctrl_t* pos) { + ctrl = _mm_loadu_si128(reinterpret_cast<const __m128i*>(pos)); + } + + // Returns a bitmask representing the positions of slots that match hash. + BitMask<uint32_t, kWidth> Match(h2_t hash) const { + auto match = _mm_set1_epi8(hash); + return BitMask<uint32_t, kWidth>( + _mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl))); + } + + // Returns a bitmask representing the positions of empty slots. + BitMask<uint32_t, kWidth> MatchEmpty() const { #if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 // This only works because ctrl_t::kEmpty is -128. - return BitMask<uint32_t, kWidth>( - _mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl))); -#else + return BitMask<uint32_t, kWidth>( + _mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl))); +#else return Match(static_cast<h2_t>(ctrl_t::kEmpty)); -#endif - } - - // Returns a bitmask representing the positions of empty or deleted slots. - BitMask<uint32_t, kWidth> MatchEmptyOrDeleted() const { +#endif + } + + // Returns a bitmask representing the positions of empty or deleted slots. + BitMask<uint32_t, kWidth> MatchEmptyOrDeleted() const { auto special = _mm_set1_epi8(static_cast<int8_t>(ctrl_t::kSentinel)); - return BitMask<uint32_t, kWidth>( - _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl))); - } - - // Returns the number of trailing empty or deleted elements in the group. - uint32_t CountLeadingEmptyOrDeleted() const { + return BitMask<uint32_t, kWidth>( + _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl))); + } + + // Returns the number of trailing empty or deleted elements in the group. + uint32_t CountLeadingEmptyOrDeleted() const { auto special = _mm_set1_epi8(static_cast<int8_t>(ctrl_t::kSentinel)); return TrailingZeros(static_cast<uint32_t>( _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)) + 1)); - } - - void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { - auto msbs = _mm_set1_epi8(static_cast<char>(-128)); - auto x126 = _mm_set1_epi8(126); + } + + void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { + auto msbs = _mm_set1_epi8(static_cast<char>(-128)); + auto x126 = _mm_set1_epi8(126); #if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 - auto res = _mm_or_si128(_mm_shuffle_epi8(x126, ctrl), msbs); -#else - auto zero = _mm_setzero_si128(); - auto special_mask = _mm_cmpgt_epi8_fixed(zero, ctrl); - auto res = _mm_or_si128(msbs, _mm_andnot_si128(special_mask, x126)); -#endif - _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), res); - } - - __m128i ctrl; -}; + auto res = _mm_or_si128(_mm_shuffle_epi8(x126, ctrl), msbs); +#else + auto zero = _mm_setzero_si128(); + auto special_mask = _mm_cmpgt_epi8_fixed(zero, ctrl); + auto res = _mm_or_si128(msbs, _mm_andnot_si128(special_mask, x126)); +#endif + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), res); + } + + __m128i ctrl; +}; #endif // ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 - -struct GroupPortableImpl { - static constexpr size_t kWidth = 8; - - explicit GroupPortableImpl(const ctrl_t* pos) - : ctrl(little_endian::Load64(pos)) {} - - BitMask<uint64_t, kWidth, 3> Match(h2_t hash) const { - // For the technique, see: - // http://graphics.stanford.edu/~seander/bithacks.html##ValueInWord - // (Determine if a word has a byte equal to n). - // - // Caveat: there are false positives but: - // - they only occur if there is a real match + +struct GroupPortableImpl { + static constexpr size_t kWidth = 8; + + explicit GroupPortableImpl(const ctrl_t* pos) + : ctrl(little_endian::Load64(pos)) {} + + BitMask<uint64_t, kWidth, 3> Match(h2_t hash) const { + // For the technique, see: + // http://graphics.stanford.edu/~seander/bithacks.html##ValueInWord + // (Determine if a word has a byte equal to n). + // + // Caveat: there are false positives but: + // - they only occur if there is a real match // - they never occur on ctrl_t::kEmpty, ctrl_t::kDeleted, ctrl_t::kSentinel - // - they will be handled gracefully by subsequent checks in code - // - // Example: - // v = 0x1716151413121110 - // hash = 0x12 - // retval = (v - lsbs) & ~v & msbs = 0x0000000080800000 - constexpr uint64_t msbs = 0x8080808080808080ULL; - constexpr uint64_t lsbs = 0x0101010101010101ULL; - auto x = ctrl ^ (lsbs * hash); - return BitMask<uint64_t, kWidth, 3>((x - lsbs) & ~x & msbs); - } - - BitMask<uint64_t, kWidth, 3> MatchEmpty() const { - constexpr uint64_t msbs = 0x8080808080808080ULL; - return BitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 6)) & msbs); - } - - BitMask<uint64_t, kWidth, 3> MatchEmptyOrDeleted() const { - constexpr uint64_t msbs = 0x8080808080808080ULL; - return BitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 7)) & msbs); - } - - uint32_t CountLeadingEmptyOrDeleted() const { - constexpr uint64_t gaps = 0x00FEFEFEFEFEFEFEULL; - return (TrailingZeros(((~ctrl & (ctrl >> 7)) | gaps) + 1) + 7) >> 3; - } - - void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { - constexpr uint64_t msbs = 0x8080808080808080ULL; - constexpr uint64_t lsbs = 0x0101010101010101ULL; - auto x = ctrl & msbs; - auto res = (~x + (x >> 7)) & ~lsbs; - little_endian::Store64(dst, res); - } - - uint64_t ctrl; -}; - + // - they will be handled gracefully by subsequent checks in code + // + // Example: + // v = 0x1716151413121110 + // hash = 0x12 + // retval = (v - lsbs) & ~v & msbs = 0x0000000080800000 + constexpr uint64_t msbs = 0x8080808080808080ULL; + constexpr uint64_t lsbs = 0x0101010101010101ULL; + auto x = ctrl ^ (lsbs * hash); + return BitMask<uint64_t, kWidth, 3>((x - lsbs) & ~x & msbs); + } + + BitMask<uint64_t, kWidth, 3> MatchEmpty() const { + constexpr uint64_t msbs = 0x8080808080808080ULL; + return BitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 6)) & msbs); + } + + BitMask<uint64_t, kWidth, 3> MatchEmptyOrDeleted() const { + constexpr uint64_t msbs = 0x8080808080808080ULL; + return BitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 7)) & msbs); + } + + uint32_t CountLeadingEmptyOrDeleted() const { + constexpr uint64_t gaps = 0x00FEFEFEFEFEFEFEULL; + return (TrailingZeros(((~ctrl & (ctrl >> 7)) | gaps) + 1) + 7) >> 3; + } + + void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { + constexpr uint64_t msbs = 0x8080808080808080ULL; + constexpr uint64_t lsbs = 0x0101010101010101ULL; + auto x = ctrl & msbs; + auto res = (~x + (x >> 7)) & ~lsbs; + little_endian::Store64(dst, res); + } + + uint64_t ctrl; +}; + #if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 -using Group = GroupSse2Impl; -#else -using Group = GroupPortableImpl; -#endif - +using Group = GroupSse2Impl; +#else +using Group = GroupPortableImpl; +#endif + // The number of cloned control bytes that we copy from the beginning to the // end of the control bytes array. constexpr size_t NumClonedBytes() { return Group::kWidth - 1; } -template <class Policy, class Hash, class Eq, class Alloc> -class raw_hash_set; - -inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; } - -// PRECONDITION: -// IsValidCapacity(capacity) +template <class Policy, class Hash, class Eq, class Alloc> +class raw_hash_set; + +inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; } + +// PRECONDITION: +// IsValidCapacity(capacity) // ctrl[capacity] == ctrl_t::kSentinel // ctrl[i] != ctrl_t::kSentinel for all i < capacity -// Applies mapping for every byte in ctrl: -// DELETED -> EMPTY -// EMPTY -> EMPTY -// FULL -> DELETED +// Applies mapping for every byte in ctrl: +// DELETED -> EMPTY +// EMPTY -> EMPTY +// FULL -> DELETED void ConvertDeletedToEmptyAndFullToDeleted(ctrl_t* ctrl, size_t capacity); - -// Rounds up the capacity to the next power of 2 minus 1, with a minimum of 1. -inline size_t NormalizeCapacity(size_t n) { + +// Rounds up the capacity to the next power of 2 minus 1, with a minimum of 1. +inline size_t NormalizeCapacity(size_t n) { return n ? ~size_t{} >> countl_zero(n) : 1; -} - +} + // General notes on capacity/growth methods below: // - We use 7/8th as maximum load factor. For 16-wide groups, that gives an // average of two empty slots per group. @@ -497,26 +497,26 @@ inline size_t NormalizeCapacity(size_t n) { // Given `capacity` of the table, returns the size (i.e. number of full slots) // at which we should grow the capacity. -inline size_t CapacityToGrowth(size_t capacity) { - assert(IsValidCapacity(capacity)); - // `capacity*7/8` - if (Group::kWidth == 8 && capacity == 7) { - // x-x/8 does not work when x==7. - return 6; - } - return capacity - capacity / 8; -} -// From desired "growth" to a lowerbound of the necessary capacity. +inline size_t CapacityToGrowth(size_t capacity) { + assert(IsValidCapacity(capacity)); + // `capacity*7/8` + if (Group::kWidth == 8 && capacity == 7) { + // x-x/8 does not work when x==7. + return 6; + } + return capacity - capacity / 8; +} +// From desired "growth" to a lowerbound of the necessary capacity. // Might not be a valid one and requires NormalizeCapacity(). -inline size_t GrowthToLowerboundCapacity(size_t growth) { - // `growth*8/7` - if (Group::kWidth == 8 && growth == 7) { - // x+(x-1)/7 does not work when x==7. - return 8; - } - return growth + static_cast<size_t>((static_cast<int64_t>(growth) - 1) / 7); -} - +inline size_t GrowthToLowerboundCapacity(size_t growth) { + // `growth*8/7` + if (Group::kWidth == 8 && growth == 7) { + // x+(x-1)/7 does not work when x==7. + return 8; + } + return growth + static_cast<size_t>((static_cast<int64_t>(growth) - 1) / 7); +} + template <class InputIter> size_t SelectBucketCountForIterRange(InputIter first, InputIter last, size_t bucket_count) { @@ -655,589 +655,589 @@ inline size_t AllocSize(size_t capacity, size_t slot_size, size_t slot_align) { return SlotOffset(capacity, slot_align) + capacity * slot_size; } -// Policy: a policy defines how to perform different operations on -// the slots of the hashtable (see hash_policy_traits.h for the full interface -// of policy). -// -// Hash: a (possibly polymorphic) functor that hashes keys of the hashtable. The -// functor should accept a key and return size_t as hash. For best performance -// it is important that the hash function provides high entropy across all bits -// of the hash. -// -// Eq: a (possibly polymorphic) functor that compares two keys for equality. It -// should accept two (of possibly different type) keys and return a bool: true -// if they are equal, false if they are not. If two keys compare equal, then -// their hash values as defined by Hash MUST be equal. -// +// Policy: a policy defines how to perform different operations on +// the slots of the hashtable (see hash_policy_traits.h for the full interface +// of policy). +// +// Hash: a (possibly polymorphic) functor that hashes keys of the hashtable. The +// functor should accept a key and return size_t as hash. For best performance +// it is important that the hash function provides high entropy across all bits +// of the hash. +// +// Eq: a (possibly polymorphic) functor that compares two keys for equality. It +// should accept two (of possibly different type) keys and return a bool: true +// if they are equal, false if they are not. If two keys compare equal, then +// their hash values as defined by Hash MUST be equal. +// // Allocator: an Allocator // [https://en.cppreference.com/w/cpp/named_req/Allocator] with which -// the storage of the hashtable will be allocated and the elements will be -// constructed and destroyed. -template <class Policy, class Hash, class Eq, class Alloc> -class raw_hash_set { - using PolicyTraits = hash_policy_traits<Policy>; - using KeyArgImpl = - KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>; - - public: - using init_type = typename PolicyTraits::init_type; - using key_type = typename PolicyTraits::key_type; - // TODO(sbenza): Hide slot_type as it is an implementation detail. Needs user - // code fixes! - using slot_type = typename PolicyTraits::slot_type; - using allocator_type = Alloc; - using size_type = size_t; - using difference_type = ptrdiff_t; - using hasher = Hash; - using key_equal = Eq; - using policy_type = Policy; - using value_type = typename PolicyTraits::value_type; - using reference = value_type&; - using const_reference = const value_type&; - using pointer = typename absl::allocator_traits< - allocator_type>::template rebind_traits<value_type>::pointer; - using const_pointer = typename absl::allocator_traits< - allocator_type>::template rebind_traits<value_type>::const_pointer; - - // Alias used for heterogeneous lookup functions. - // `key_arg<K>` evaluates to `K` when the functors are transparent and to - // `key_type` otherwise. It permits template argument deduction on `K` for the - // transparent case. - template <class K> - using key_arg = typename KeyArgImpl::template type<K, key_type>; - - private: - // Give an early error when key_type is not hashable/eq. - auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); - auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); - - using AllocTraits = absl::allocator_traits<allocator_type>; - using SlotAlloc = typename absl::allocator_traits< - allocator_type>::template rebind_alloc<slot_type>; - using SlotAllocTraits = typename absl::allocator_traits< - allocator_type>::template rebind_traits<slot_type>; - - static_assert(std::is_lvalue_reference<reference>::value, - "Policy::element() must return a reference"); - - template <typename T> - struct SameAsElementReference - : std::is_same<typename std::remove_cv< - typename std::remove_reference<reference>::type>::type, - typename std::remove_cv< - typename std::remove_reference<T>::type>::type> {}; - - // An enabler for insert(T&&): T must be convertible to init_type or be the - // same as [cv] value_type [ref]. - // Note: we separate SameAsElementReference into its own type to avoid using - // reference unless we need to. MSVC doesn't seem to like it in some - // cases. - template <class T> - using RequiresInsertable = typename std::enable_if< - absl::disjunction<std::is_convertible<T, init_type>, - SameAsElementReference<T>>::value, - int>::type; - - // RequiresNotInit is a workaround for gcc prior to 7.1. - // See https://godbolt.org/g/Y4xsUh. - template <class T> - using RequiresNotInit = - typename std::enable_if<!std::is_same<T, init_type>::value, int>::type; - - template <class... Ts> - using IsDecomposable = IsDecomposable<void, PolicyTraits, Hash, Eq, Ts...>; - - public: - static_assert(std::is_same<pointer, value_type*>::value, - "Allocators with custom pointer types are not supported"); - static_assert(std::is_same<const_pointer, const value_type*>::value, - "Allocators with custom pointer types are not supported"); - - class iterator { - friend class raw_hash_set; - - public: - using iterator_category = std::forward_iterator_tag; - using value_type = typename raw_hash_set::value_type; - using reference = - absl::conditional_t<PolicyTraits::constant_iterators::value, - const value_type&, value_type&>; - using pointer = absl::remove_reference_t<reference>*; - using difference_type = typename raw_hash_set::difference_type; - - iterator() {} - - // PRECONDITION: not an end() iterator. - reference operator*() const { +// the storage of the hashtable will be allocated and the elements will be +// constructed and destroyed. +template <class Policy, class Hash, class Eq, class Alloc> +class raw_hash_set { + using PolicyTraits = hash_policy_traits<Policy>; + using KeyArgImpl = + KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>; + + public: + using init_type = typename PolicyTraits::init_type; + using key_type = typename PolicyTraits::key_type; + // TODO(sbenza): Hide slot_type as it is an implementation detail. Needs user + // code fixes! + using slot_type = typename PolicyTraits::slot_type; + using allocator_type = Alloc; + using size_type = size_t; + using difference_type = ptrdiff_t; + using hasher = Hash; + using key_equal = Eq; + using policy_type = Policy; + using value_type = typename PolicyTraits::value_type; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = typename absl::allocator_traits< + allocator_type>::template rebind_traits<value_type>::pointer; + using const_pointer = typename absl::allocator_traits< + allocator_type>::template rebind_traits<value_type>::const_pointer; + + // Alias used for heterogeneous lookup functions. + // `key_arg<K>` evaluates to `K` when the functors are transparent and to + // `key_type` otherwise. It permits template argument deduction on `K` for the + // transparent case. + template <class K> + using key_arg = typename KeyArgImpl::template type<K, key_type>; + + private: + // Give an early error when key_type is not hashable/eq. + auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); + auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); + + using AllocTraits = absl::allocator_traits<allocator_type>; + using SlotAlloc = typename absl::allocator_traits< + allocator_type>::template rebind_alloc<slot_type>; + using SlotAllocTraits = typename absl::allocator_traits< + allocator_type>::template rebind_traits<slot_type>; + + static_assert(std::is_lvalue_reference<reference>::value, + "Policy::element() must return a reference"); + + template <typename T> + struct SameAsElementReference + : std::is_same<typename std::remove_cv< + typename std::remove_reference<reference>::type>::type, + typename std::remove_cv< + typename std::remove_reference<T>::type>::type> {}; + + // An enabler for insert(T&&): T must be convertible to init_type or be the + // same as [cv] value_type [ref]. + // Note: we separate SameAsElementReference into its own type to avoid using + // reference unless we need to. MSVC doesn't seem to like it in some + // cases. + template <class T> + using RequiresInsertable = typename std::enable_if< + absl::disjunction<std::is_convertible<T, init_type>, + SameAsElementReference<T>>::value, + int>::type; + + // RequiresNotInit is a workaround for gcc prior to 7.1. + // See https://godbolt.org/g/Y4xsUh. + template <class T> + using RequiresNotInit = + typename std::enable_if<!std::is_same<T, init_type>::value, int>::type; + + template <class... Ts> + using IsDecomposable = IsDecomposable<void, PolicyTraits, Hash, Eq, Ts...>; + + public: + static_assert(std::is_same<pointer, value_type*>::value, + "Allocators with custom pointer types are not supported"); + static_assert(std::is_same<const_pointer, const value_type*>::value, + "Allocators with custom pointer types are not supported"); + + class iterator { + friend class raw_hash_set; + + public: + using iterator_category = std::forward_iterator_tag; + using value_type = typename raw_hash_set::value_type; + using reference = + absl::conditional_t<PolicyTraits::constant_iterators::value, + const value_type&, value_type&>; + using pointer = absl::remove_reference_t<reference>*; + using difference_type = typename raw_hash_set::difference_type; + + iterator() {} + + // PRECONDITION: not an end() iterator. + reference operator*() const { AssertIsFull(ctrl_); - return PolicyTraits::element(slot_); - } - - // PRECONDITION: not an end() iterator. - pointer operator->() const { return &operator*(); } - - // PRECONDITION: not an end() iterator. - iterator& operator++() { + return PolicyTraits::element(slot_); + } + + // PRECONDITION: not an end() iterator. + pointer operator->() const { return &operator*(); } + + // PRECONDITION: not an end() iterator. + iterator& operator++() { AssertIsFull(ctrl_); - ++ctrl_; - ++slot_; - skip_empty_or_deleted(); - return *this; - } - // PRECONDITION: not an end() iterator. - iterator operator++(int) { - auto tmp = *this; - ++*this; - return tmp; - } - - friend bool operator==(const iterator& a, const iterator& b) { + ++ctrl_; + ++slot_; + skip_empty_or_deleted(); + return *this; + } + // PRECONDITION: not an end() iterator. + iterator operator++(int) { + auto tmp = *this; + ++*this; + return tmp; + } + + friend bool operator==(const iterator& a, const iterator& b) { AssertIsValid(a.ctrl_); AssertIsValid(b.ctrl_); - return a.ctrl_ == b.ctrl_; - } - friend bool operator!=(const iterator& a, const iterator& b) { - return !(a == b); - } - - private: + return a.ctrl_ == b.ctrl_; + } + friend bool operator!=(const iterator& a, const iterator& b) { + return !(a == b); + } + + private: iterator(ctrl_t* ctrl, slot_type* slot) : ctrl_(ctrl), slot_(slot) { // This assumption helps the compiler know that any non-end iterator is // not equal to any end iterator. ABSL_INTERNAL_ASSUME(ctrl != nullptr); - } - - void skip_empty_or_deleted() { - while (IsEmptyOrDeleted(*ctrl_)) { - uint32_t shift = Group{ctrl_}.CountLeadingEmptyOrDeleted(); - ctrl_ += shift; - slot_ += shift; - } + } + + void skip_empty_or_deleted() { + while (IsEmptyOrDeleted(*ctrl_)) { + uint32_t shift = Group{ctrl_}.CountLeadingEmptyOrDeleted(); + ctrl_ += shift; + slot_ += shift; + } if (ABSL_PREDICT_FALSE(*ctrl_ == ctrl_t::kSentinel)) ctrl_ = nullptr; - } - - ctrl_t* ctrl_ = nullptr; - // To avoid uninitialized member warnings, put slot_ in an anonymous union. - // The member is not initialized on singleton and end iterators. - union { - slot_type* slot_; - }; - }; - - class const_iterator { - friend class raw_hash_set; - - public: - using iterator_category = typename iterator::iterator_category; - using value_type = typename raw_hash_set::value_type; - using reference = typename raw_hash_set::const_reference; - using pointer = typename raw_hash_set::const_pointer; - using difference_type = typename raw_hash_set::difference_type; - - const_iterator() {} - // Implicit construction from iterator. - const_iterator(iterator i) : inner_(std::move(i)) {} - - reference operator*() const { return *inner_; } - pointer operator->() const { return inner_.operator->(); } - - const_iterator& operator++() { - ++inner_; - return *this; - } - const_iterator operator++(int) { return inner_++; } - - friend bool operator==(const const_iterator& a, const const_iterator& b) { - return a.inner_ == b.inner_; - } - friend bool operator!=(const const_iterator& a, const const_iterator& b) { - return !(a == b); - } - - private: - const_iterator(const ctrl_t* ctrl, const slot_type* slot) - : inner_(const_cast<ctrl_t*>(ctrl), const_cast<slot_type*>(slot)) {} - - iterator inner_; - }; - - using node_type = node_handle<Policy, hash_policy_traits<Policy>, Alloc>; - using insert_return_type = InsertReturnType<iterator, node_type>; - - raw_hash_set() noexcept( - std::is_nothrow_default_constructible<hasher>::value&& - std::is_nothrow_default_constructible<key_equal>::value&& - std::is_nothrow_default_constructible<allocator_type>::value) {} - - explicit raw_hash_set(size_t bucket_count, const hasher& hash = hasher(), - const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) + } + + ctrl_t* ctrl_ = nullptr; + // To avoid uninitialized member warnings, put slot_ in an anonymous union. + // The member is not initialized on singleton and end iterators. + union { + slot_type* slot_; + }; + }; + + class const_iterator { + friend class raw_hash_set; + + public: + using iterator_category = typename iterator::iterator_category; + using value_type = typename raw_hash_set::value_type; + using reference = typename raw_hash_set::const_reference; + using pointer = typename raw_hash_set::const_pointer; + using difference_type = typename raw_hash_set::difference_type; + + const_iterator() {} + // Implicit construction from iterator. + const_iterator(iterator i) : inner_(std::move(i)) {} + + reference operator*() const { return *inner_; } + pointer operator->() const { return inner_.operator->(); } + + const_iterator& operator++() { + ++inner_; + return *this; + } + const_iterator operator++(int) { return inner_++; } + + friend bool operator==(const const_iterator& a, const const_iterator& b) { + return a.inner_ == b.inner_; + } + friend bool operator!=(const const_iterator& a, const const_iterator& b) { + return !(a == b); + } + + private: + const_iterator(const ctrl_t* ctrl, const slot_type* slot) + : inner_(const_cast<ctrl_t*>(ctrl), const_cast<slot_type*>(slot)) {} + + iterator inner_; + }; + + using node_type = node_handle<Policy, hash_policy_traits<Policy>, Alloc>; + using insert_return_type = InsertReturnType<iterator, node_type>; + + raw_hash_set() noexcept( + std::is_nothrow_default_constructible<hasher>::value&& + std::is_nothrow_default_constructible<key_equal>::value&& + std::is_nothrow_default_constructible<allocator_type>::value) {} + + explicit raw_hash_set(size_t bucket_count, const hasher& hash = hasher(), + const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) : ctrl_(EmptyGroup()), settings_(0, HashtablezInfoHandle(), hash, eq, alloc) { - if (bucket_count) { - capacity_ = NormalizeCapacity(bucket_count); - initialize_slots(); - } - } - - raw_hash_set(size_t bucket_count, const hasher& hash, - const allocator_type& alloc) - : raw_hash_set(bucket_count, hash, key_equal(), alloc) {} - - raw_hash_set(size_t bucket_count, const allocator_type& alloc) - : raw_hash_set(bucket_count, hasher(), key_equal(), alloc) {} - - explicit raw_hash_set(const allocator_type& alloc) - : raw_hash_set(0, hasher(), key_equal(), alloc) {} - - template <class InputIter> - raw_hash_set(InputIter first, InputIter last, size_t bucket_count = 0, - const hasher& hash = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) + if (bucket_count) { + capacity_ = NormalizeCapacity(bucket_count); + initialize_slots(); + } + } + + raw_hash_set(size_t bucket_count, const hasher& hash, + const allocator_type& alloc) + : raw_hash_set(bucket_count, hash, key_equal(), alloc) {} + + raw_hash_set(size_t bucket_count, const allocator_type& alloc) + : raw_hash_set(bucket_count, hasher(), key_equal(), alloc) {} + + explicit raw_hash_set(const allocator_type& alloc) + : raw_hash_set(0, hasher(), key_equal(), alloc) {} + + template <class InputIter> + raw_hash_set(InputIter first, InputIter last, size_t bucket_count = 0, + const hasher& hash = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) : raw_hash_set(SelectBucketCountForIterRange(first, last, bucket_count), hash, eq, alloc) { - insert(first, last); - } - - template <class InputIter> - raw_hash_set(InputIter first, InputIter last, size_t bucket_count, - const hasher& hash, const allocator_type& alloc) - : raw_hash_set(first, last, bucket_count, hash, key_equal(), alloc) {} - - template <class InputIter> - raw_hash_set(InputIter first, InputIter last, size_t bucket_count, - const allocator_type& alloc) - : raw_hash_set(first, last, bucket_count, hasher(), key_equal(), alloc) {} - - template <class InputIter> - raw_hash_set(InputIter first, InputIter last, const allocator_type& alloc) - : raw_hash_set(first, last, 0, hasher(), key_equal(), alloc) {} - - // Instead of accepting std::initializer_list<value_type> as the first - // argument like std::unordered_set<value_type> does, we have two overloads - // that accept std::initializer_list<T> and std::initializer_list<init_type>. - // This is advantageous for performance. - // - // // Turns {"abc", "def"} into std::initializer_list<std::string>, then - // // copies the strings into the set. - // std::unordered_set<std::string> s = {"abc", "def"}; - // - // // Turns {"abc", "def"} into std::initializer_list<const char*>, then - // // copies the strings into the set. - // absl::flat_hash_set<std::string> s = {"abc", "def"}; - // - // The same trick is used in insert(). - // - // The enabler is necessary to prevent this constructor from triggering where - // the copy constructor is meant to be called. - // - // absl::flat_hash_set<int> a, b{a}; - // - // RequiresNotInit<T> is a workaround for gcc prior to 7.1. - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - raw_hash_set(std::initializer_list<T> init, size_t bucket_count = 0, - const hasher& hash = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : raw_hash_set(init.begin(), init.end(), bucket_count, hash, eq, alloc) {} - - raw_hash_set(std::initializer_list<init_type> init, size_t bucket_count = 0, - const hasher& hash = hasher(), const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) - : raw_hash_set(init.begin(), init.end(), bucket_count, hash, eq, alloc) {} - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - raw_hash_set(std::initializer_list<T> init, size_t bucket_count, - const hasher& hash, const allocator_type& alloc) - : raw_hash_set(init, bucket_count, hash, key_equal(), alloc) {} - - raw_hash_set(std::initializer_list<init_type> init, size_t bucket_count, - const hasher& hash, const allocator_type& alloc) - : raw_hash_set(init, bucket_count, hash, key_equal(), alloc) {} - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - raw_hash_set(std::initializer_list<T> init, size_t bucket_count, - const allocator_type& alloc) - : raw_hash_set(init, bucket_count, hasher(), key_equal(), alloc) {} - - raw_hash_set(std::initializer_list<init_type> init, size_t bucket_count, - const allocator_type& alloc) - : raw_hash_set(init, bucket_count, hasher(), key_equal(), alloc) {} - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> - raw_hash_set(std::initializer_list<T> init, const allocator_type& alloc) - : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} - - raw_hash_set(std::initializer_list<init_type> init, - const allocator_type& alloc) - : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} - - raw_hash_set(const raw_hash_set& that) - : raw_hash_set(that, AllocTraits::select_on_container_copy_construction( - that.alloc_ref())) {} - - raw_hash_set(const raw_hash_set& that, const allocator_type& a) - : raw_hash_set(0, that.hash_ref(), that.eq_ref(), a) { - reserve(that.size()); - // Because the table is guaranteed to be empty, we can do something faster - // than a full `insert`. - for (const auto& v : that) { - const size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, v); + insert(first, last); + } + + template <class InputIter> + raw_hash_set(InputIter first, InputIter last, size_t bucket_count, + const hasher& hash, const allocator_type& alloc) + : raw_hash_set(first, last, bucket_count, hash, key_equal(), alloc) {} + + template <class InputIter> + raw_hash_set(InputIter first, InputIter last, size_t bucket_count, + const allocator_type& alloc) + : raw_hash_set(first, last, bucket_count, hasher(), key_equal(), alloc) {} + + template <class InputIter> + raw_hash_set(InputIter first, InputIter last, const allocator_type& alloc) + : raw_hash_set(first, last, 0, hasher(), key_equal(), alloc) {} + + // Instead of accepting std::initializer_list<value_type> as the first + // argument like std::unordered_set<value_type> does, we have two overloads + // that accept std::initializer_list<T> and std::initializer_list<init_type>. + // This is advantageous for performance. + // + // // Turns {"abc", "def"} into std::initializer_list<std::string>, then + // // copies the strings into the set. + // std::unordered_set<std::string> s = {"abc", "def"}; + // + // // Turns {"abc", "def"} into std::initializer_list<const char*>, then + // // copies the strings into the set. + // absl::flat_hash_set<std::string> s = {"abc", "def"}; + // + // The same trick is used in insert(). + // + // The enabler is necessary to prevent this constructor from triggering where + // the copy constructor is meant to be called. + // + // absl::flat_hash_set<int> a, b{a}; + // + // RequiresNotInit<T> is a workaround for gcc prior to 7.1. + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> + raw_hash_set(std::initializer_list<T> init, size_t bucket_count = 0, + const hasher& hash = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : raw_hash_set(init.begin(), init.end(), bucket_count, hash, eq, alloc) {} + + raw_hash_set(std::initializer_list<init_type> init, size_t bucket_count = 0, + const hasher& hash = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : raw_hash_set(init.begin(), init.end(), bucket_count, hash, eq, alloc) {} + + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> + raw_hash_set(std::initializer_list<T> init, size_t bucket_count, + const hasher& hash, const allocator_type& alloc) + : raw_hash_set(init, bucket_count, hash, key_equal(), alloc) {} + + raw_hash_set(std::initializer_list<init_type> init, size_t bucket_count, + const hasher& hash, const allocator_type& alloc) + : raw_hash_set(init, bucket_count, hash, key_equal(), alloc) {} + + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> + raw_hash_set(std::initializer_list<T> init, size_t bucket_count, + const allocator_type& alloc) + : raw_hash_set(init, bucket_count, hasher(), key_equal(), alloc) {} + + raw_hash_set(std::initializer_list<init_type> init, size_t bucket_count, + const allocator_type& alloc) + : raw_hash_set(init, bucket_count, hasher(), key_equal(), alloc) {} + + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> + raw_hash_set(std::initializer_list<T> init, const allocator_type& alloc) + : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} + + raw_hash_set(std::initializer_list<init_type> init, + const allocator_type& alloc) + : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} + + raw_hash_set(const raw_hash_set& that) + : raw_hash_set(that, AllocTraits::select_on_container_copy_construction( + that.alloc_ref())) {} + + raw_hash_set(const raw_hash_set& that, const allocator_type& a) + : raw_hash_set(0, that.hash_ref(), that.eq_ref(), a) { + reserve(that.size()); + // Because the table is guaranteed to be empty, we can do something faster + // than a full `insert`. + for (const auto& v : that) { + const size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, v); auto target = find_first_non_full(ctrl_, hash, capacity_); SetCtrl(target.offset, H2(hash), capacity_, ctrl_, slots_, sizeof(slot_type)); - emplace_at(target.offset, v); + emplace_at(target.offset, v); infoz().RecordInsert(hash, target.probe_length); - } - size_ = that.size(); - growth_left() -= that.size(); - } - - raw_hash_set(raw_hash_set&& that) noexcept( - std::is_nothrow_copy_constructible<hasher>::value&& - std::is_nothrow_copy_constructible<key_equal>::value&& - std::is_nothrow_copy_constructible<allocator_type>::value) - : ctrl_(absl::exchange(that.ctrl_, EmptyGroup())), - slots_(absl::exchange(that.slots_, nullptr)), - size_(absl::exchange(that.size_, 0)), - capacity_(absl::exchange(that.capacity_, 0)), - // Hash, equality and allocator are copied instead of moved because - // `that` must be left valid. If Hash is std::function<Key>, moving it - // would create a nullptr functor that cannot be called. + } + size_ = that.size(); + growth_left() -= that.size(); + } + + raw_hash_set(raw_hash_set&& that) noexcept( + std::is_nothrow_copy_constructible<hasher>::value&& + std::is_nothrow_copy_constructible<key_equal>::value&& + std::is_nothrow_copy_constructible<allocator_type>::value) + : ctrl_(absl::exchange(that.ctrl_, EmptyGroup())), + slots_(absl::exchange(that.slots_, nullptr)), + size_(absl::exchange(that.size_, 0)), + capacity_(absl::exchange(that.capacity_, 0)), + // Hash, equality and allocator are copied instead of moved because + // `that` must be left valid. If Hash is std::function<Key>, moving it + // would create a nullptr functor that cannot be called. settings_(absl::exchange(that.growth_left(), 0), absl::exchange(that.infoz(), HashtablezInfoHandle()), that.hash_ref(), that.eq_ref(), that.alloc_ref()) {} - - raw_hash_set(raw_hash_set&& that, const allocator_type& a) - : ctrl_(EmptyGroup()), - slots_(nullptr), - size_(0), - capacity_(0), + + raw_hash_set(raw_hash_set&& that, const allocator_type& a) + : ctrl_(EmptyGroup()), + slots_(nullptr), + size_(0), + capacity_(0), settings_(0, HashtablezInfoHandle(), that.hash_ref(), that.eq_ref(), a) { - if (a == that.alloc_ref()) { - std::swap(ctrl_, that.ctrl_); - std::swap(slots_, that.slots_); - std::swap(size_, that.size_); - std::swap(capacity_, that.capacity_); - std::swap(growth_left(), that.growth_left()); + if (a == that.alloc_ref()) { + std::swap(ctrl_, that.ctrl_); + std::swap(slots_, that.slots_); + std::swap(size_, that.size_); + std::swap(capacity_, that.capacity_); + std::swap(growth_left(), that.growth_left()); std::swap(infoz(), that.infoz()); - } else { - reserve(that.size()); - // Note: this will copy elements of dense_set and unordered_set instead of - // moving them. This can be fixed if it ever becomes an issue. - for (auto& elem : that) insert(std::move(elem)); - } - } - - raw_hash_set& operator=(const raw_hash_set& that) { - raw_hash_set tmp(that, - AllocTraits::propagate_on_container_copy_assignment::value - ? that.alloc_ref() - : alloc_ref()); - swap(tmp); - return *this; - } - - raw_hash_set& operator=(raw_hash_set&& that) noexcept( - absl::allocator_traits<allocator_type>::is_always_equal::value&& - std::is_nothrow_move_assignable<hasher>::value&& - std::is_nothrow_move_assignable<key_equal>::value) { - // TODO(sbenza): We should only use the operations from the noexcept clause - // to make sure we actually adhere to that contract. - return move_assign( - std::move(that), - typename AllocTraits::propagate_on_container_move_assignment()); - } - - ~raw_hash_set() { destroy_slots(); } - - iterator begin() { - auto it = iterator_at(0); - it.skip_empty_or_deleted(); - return it; - } + } else { + reserve(that.size()); + // Note: this will copy elements of dense_set and unordered_set instead of + // moving them. This can be fixed if it ever becomes an issue. + for (auto& elem : that) insert(std::move(elem)); + } + } + + raw_hash_set& operator=(const raw_hash_set& that) { + raw_hash_set tmp(that, + AllocTraits::propagate_on_container_copy_assignment::value + ? that.alloc_ref() + : alloc_ref()); + swap(tmp); + return *this; + } + + raw_hash_set& operator=(raw_hash_set&& that) noexcept( + absl::allocator_traits<allocator_type>::is_always_equal::value&& + std::is_nothrow_move_assignable<hasher>::value&& + std::is_nothrow_move_assignable<key_equal>::value) { + // TODO(sbenza): We should only use the operations from the noexcept clause + // to make sure we actually adhere to that contract. + return move_assign( + std::move(that), + typename AllocTraits::propagate_on_container_move_assignment()); + } + + ~raw_hash_set() { destroy_slots(); } + + iterator begin() { + auto it = iterator_at(0); + it.skip_empty_or_deleted(); + return it; + } iterator end() { return {}; } - - const_iterator begin() const { - return const_cast<raw_hash_set*>(this)->begin(); - } + + const_iterator begin() const { + return const_cast<raw_hash_set*>(this)->begin(); + } const_iterator end() const { return {}; } - const_iterator cbegin() const { return begin(); } - const_iterator cend() const { return end(); } - - bool empty() const { return !size(); } - size_t size() const { return size_; } - size_t capacity() const { return capacity_; } - size_t max_size() const { return (std::numeric_limits<size_t>::max)(); } - - ABSL_ATTRIBUTE_REINITIALIZES void clear() { - // Iterating over this container is O(bucket_count()). When bucket_count() - // is much greater than size(), iteration becomes prohibitively expensive. - // For clear() it is more important to reuse the allocated array when the - // container is small because allocation takes comparatively long time - // compared to destruction of the elements of the container. So we pick the - // largest bucket_count() threshold for which iteration is still fast and - // past that we simply deallocate the array. - if (capacity_ > 127) { - destroy_slots(); + const_iterator cbegin() const { return begin(); } + const_iterator cend() const { return end(); } + + bool empty() const { return !size(); } + size_t size() const { return size_; } + size_t capacity() const { return capacity_; } + size_t max_size() const { return (std::numeric_limits<size_t>::max)(); } + + ABSL_ATTRIBUTE_REINITIALIZES void clear() { + // Iterating over this container is O(bucket_count()). When bucket_count() + // is much greater than size(), iteration becomes prohibitively expensive. + // For clear() it is more important to reuse the allocated array when the + // container is small because allocation takes comparatively long time + // compared to destruction of the elements of the container. So we pick the + // largest bucket_count() threshold for which iteration is still fast and + // past that we simply deallocate the array. + if (capacity_ > 127) { + destroy_slots(); infoz().RecordClearedReservation(); - } else if (capacity_) { - for (size_t i = 0; i != capacity_; ++i) { - if (IsFull(ctrl_[i])) { - PolicyTraits::destroy(&alloc_ref(), slots_ + i); - } - } - size_ = 0; + } else if (capacity_) { + for (size_t i = 0; i != capacity_; ++i) { + if (IsFull(ctrl_[i])) { + PolicyTraits::destroy(&alloc_ref(), slots_ + i); + } + } + size_ = 0; ResetCtrl(capacity_, ctrl_, slots_, sizeof(slot_type)); - reset_growth_left(); - } - assert(empty()); + reset_growth_left(); + } + assert(empty()); infoz().RecordStorageChanged(0, capacity_); - } - - // This overload kicks in when the argument is an rvalue of insertable and - // decomposable type other than init_type. - // - // flat_hash_map<std::string, int> m; - // m.insert(std::make_pair("abc", 42)); - // TODO(cheshire): A type alias T2 is introduced as a workaround for the nvcc - // bug. - template <class T, RequiresInsertable<T> = 0, - class T2 = T, - typename std::enable_if<IsDecomposable<T2>::value, int>::type = 0, - T* = nullptr> - std::pair<iterator, bool> insert(T&& value) { - return emplace(std::forward<T>(value)); - } - - // This overload kicks in when the argument is a bitfield or an lvalue of - // insertable and decomposable type. - // - // union { int n : 1; }; - // flat_hash_set<int> s; - // s.insert(n); - // - // flat_hash_set<std::string> s; - // const char* p = "hello"; - // s.insert(p); - // - // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace - // RequiresInsertable<T> with RequiresInsertable<const T&>. - // We are hitting this bug: https://godbolt.org/g/1Vht4f. - template < - class T, RequiresInsertable<T> = 0, - typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0> - std::pair<iterator, bool> insert(const T& value) { - return emplace(value); - } - - // This overload kicks in when the argument is an rvalue of init_type. Its - // purpose is to handle brace-init-list arguments. - // - // flat_hash_map<std::string, int> s; - // s.insert({"abc", 42}); - std::pair<iterator, bool> insert(init_type&& value) { - return emplace(std::move(value)); - } - - // TODO(cheshire): A type alias T2 is introduced as a workaround for the nvcc - // bug. - template <class T, RequiresInsertable<T> = 0, class T2 = T, - typename std::enable_if<IsDecomposable<T2>::value, int>::type = 0, - T* = nullptr> - iterator insert(const_iterator, T&& value) { - return insert(std::forward<T>(value)).first; - } - - // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace - // RequiresInsertable<T> with RequiresInsertable<const T&>. - // We are hitting this bug: https://godbolt.org/g/1Vht4f. - template < - class T, RequiresInsertable<T> = 0, - typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0> - iterator insert(const_iterator, const T& value) { - return insert(value).first; - } - - iterator insert(const_iterator, init_type&& value) { - return insert(std::move(value)).first; - } - - template <class InputIt> - void insert(InputIt first, InputIt last) { + } + + // This overload kicks in when the argument is an rvalue of insertable and + // decomposable type other than init_type. + // + // flat_hash_map<std::string, int> m; + // m.insert(std::make_pair("abc", 42)); + // TODO(cheshire): A type alias T2 is introduced as a workaround for the nvcc + // bug. + template <class T, RequiresInsertable<T> = 0, + class T2 = T, + typename std::enable_if<IsDecomposable<T2>::value, int>::type = 0, + T* = nullptr> + std::pair<iterator, bool> insert(T&& value) { + return emplace(std::forward<T>(value)); + } + + // This overload kicks in when the argument is a bitfield or an lvalue of + // insertable and decomposable type. + // + // union { int n : 1; }; + // flat_hash_set<int> s; + // s.insert(n); + // + // flat_hash_set<std::string> s; + // const char* p = "hello"; + // s.insert(p); + // + // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace + // RequiresInsertable<T> with RequiresInsertable<const T&>. + // We are hitting this bug: https://godbolt.org/g/1Vht4f. + template < + class T, RequiresInsertable<T> = 0, + typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0> + std::pair<iterator, bool> insert(const T& value) { + return emplace(value); + } + + // This overload kicks in when the argument is an rvalue of init_type. Its + // purpose is to handle brace-init-list arguments. + // + // flat_hash_map<std::string, int> s; + // s.insert({"abc", 42}); + std::pair<iterator, bool> insert(init_type&& value) { + return emplace(std::move(value)); + } + + // TODO(cheshire): A type alias T2 is introduced as a workaround for the nvcc + // bug. + template <class T, RequiresInsertable<T> = 0, class T2 = T, + typename std::enable_if<IsDecomposable<T2>::value, int>::type = 0, + T* = nullptr> + iterator insert(const_iterator, T&& value) { + return insert(std::forward<T>(value)).first; + } + + // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace + // RequiresInsertable<T> with RequiresInsertable<const T&>. + // We are hitting this bug: https://godbolt.org/g/1Vht4f. + template < + class T, RequiresInsertable<T> = 0, + typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0> + iterator insert(const_iterator, const T& value) { + return insert(value).first; + } + + iterator insert(const_iterator, init_type&& value) { + return insert(std::move(value)).first; + } + + template <class InputIt> + void insert(InputIt first, InputIt last) { for (; first != last; ++first) emplace(*first); - } - - template <class T, RequiresNotInit<T> = 0, RequiresInsertable<const T&> = 0> - void insert(std::initializer_list<T> ilist) { - insert(ilist.begin(), ilist.end()); - } - - void insert(std::initializer_list<init_type> ilist) { - insert(ilist.begin(), ilist.end()); - } - - insert_return_type insert(node_type&& node) { - if (!node) return {end(), false, node_type()}; - const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node)); - auto res = PolicyTraits::apply( - InsertSlot<false>{*this, std::move(*CommonAccess::GetSlot(node))}, - elem); - if (res.second) { - CommonAccess::Reset(&node); - return {res.first, true, node_type()}; - } else { - return {res.first, false, std::move(node)}; - } - } - - iterator insert(const_iterator, node_type&& node) { + } + + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<const T&> = 0> + void insert(std::initializer_list<T> ilist) { + insert(ilist.begin(), ilist.end()); + } + + void insert(std::initializer_list<init_type> ilist) { + insert(ilist.begin(), ilist.end()); + } + + insert_return_type insert(node_type&& node) { + if (!node) return {end(), false, node_type()}; + const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node)); + auto res = PolicyTraits::apply( + InsertSlot<false>{*this, std::move(*CommonAccess::GetSlot(node))}, + elem); + if (res.second) { + CommonAccess::Reset(&node); + return {res.first, true, node_type()}; + } else { + return {res.first, false, std::move(node)}; + } + } + + iterator insert(const_iterator, node_type&& node) { auto res = insert(std::move(node)); node = std::move(res.node); return res.position; - } - - // This overload kicks in if we can deduce the key from args. This enables us - // to avoid constructing value_type if an entry with the same key already - // exists. - // - // For example: - // - // flat_hash_map<std::string, std::string> m = {{"abc", "def"}}; - // // Creates no std::string copies and makes no heap allocations. - // m.emplace("abc", "xyz"); - template <class... Args, typename std::enable_if< - IsDecomposable<Args...>::value, int>::type = 0> - std::pair<iterator, bool> emplace(Args&&... args) { - return PolicyTraits::apply(EmplaceDecomposable{*this}, - std::forward<Args>(args)...); - } - - // This overload kicks in if we cannot deduce the key from args. It constructs - // value_type unconditionally and then either moves it into the table or - // destroys. - template <class... Args, typename std::enable_if< - !IsDecomposable<Args...>::value, int>::type = 0> - std::pair<iterator, bool> emplace(Args&&... args) { + } + + // This overload kicks in if we can deduce the key from args. This enables us + // to avoid constructing value_type if an entry with the same key already + // exists. + // + // For example: + // + // flat_hash_map<std::string, std::string> m = {{"abc", "def"}}; + // // Creates no std::string copies and makes no heap allocations. + // m.emplace("abc", "xyz"); + template <class... Args, typename std::enable_if< + IsDecomposable<Args...>::value, int>::type = 0> + std::pair<iterator, bool> emplace(Args&&... args) { + return PolicyTraits::apply(EmplaceDecomposable{*this}, + std::forward<Args>(args)...); + } + + // This overload kicks in if we cannot deduce the key from args. It constructs + // value_type unconditionally and then either moves it into the table or + // destroys. + template <class... Args, typename std::enable_if< + !IsDecomposable<Args...>::value, int>::type = 0> + std::pair<iterator, bool> emplace(Args&&... args) { alignas(slot_type) unsigned char raw[sizeof(slot_type)]; - slot_type* slot = reinterpret_cast<slot_type*>(&raw); - - PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...); - const auto& elem = PolicyTraits::element(slot); - return PolicyTraits::apply(InsertSlot<true>{*this, std::move(*slot)}, elem); - } - - template <class... Args> - iterator emplace_hint(const_iterator, Args&&... args) { - return emplace(std::forward<Args>(args)...).first; - } - - // Extension API: support for lazy emplace. - // - // Looks up key in the table. If found, returns the iterator to the element. + slot_type* slot = reinterpret_cast<slot_type*>(&raw); + + PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...); + const auto& elem = PolicyTraits::element(slot); + return PolicyTraits::apply(InsertSlot<true>{*this, std::move(*slot)}, elem); + } + + template <class... Args> + iterator emplace_hint(const_iterator, Args&&... args) { + return emplace(std::forward<Args>(args)...).first; + } + + // Extension API: support for lazy emplace. + // + // Looks up key in the table. If found, returns the iterator to the element. // Otherwise calls `f` with one argument of type `raw_hash_set::constructor`. - // + // // `f` must abide by several restrictions: // - it MUST call `raw_hash_set::constructor` with arguments as if a // `raw_hash_set::value_type` is constructed, @@ -1246,172 +1246,172 @@ class raw_hash_set { // - it MUST NOT erase the lazily emplaced element. // Doing any of these is undefined behavior. // - // For example: - // - // std::unordered_set<ArenaString> s; - // // Makes ArenaStr even if "abc" is in the map. - // s.insert(ArenaString(&arena, "abc")); - // - // flat_hash_set<ArenaStr> s; - // // Makes ArenaStr only if "abc" is not in the map. - // s.lazy_emplace("abc", [&](const constructor& ctor) { - // ctor(&arena, "abc"); - // }); - // - // WARNING: This API is currently experimental. If there is a way to implement - // the same thing with the rest of the API, prefer that. - class constructor { - friend class raw_hash_set; - - public: - template <class... Args> - void operator()(Args&&... args) const { - assert(*slot_); - PolicyTraits::construct(alloc_, *slot_, std::forward<Args>(args)...); - *slot_ = nullptr; - } - - private: - constructor(allocator_type* a, slot_type** slot) : alloc_(a), slot_(slot) {} - - allocator_type* alloc_; - slot_type** slot_; - }; - - template <class K = key_type, class F> - iterator lazy_emplace(const key_arg<K>& key, F&& f) { - auto res = find_or_prepare_insert(key); - if (res.second) { - slot_type* slot = slots_ + res.first; - std::forward<F>(f)(constructor(&alloc_ref(), &slot)); - assert(!slot); - } - return iterator_at(res.first); - } - - // Extension API: support for heterogeneous keys. - // - // std::unordered_set<std::string> s; - // // Turns "abc" into std::string. - // s.erase("abc"); - // - // flat_hash_set<std::string> s; - // // Uses "abc" directly without copying it into std::string. - // s.erase("abc"); - template <class K = key_type> - size_type erase(const key_arg<K>& key) { - auto it = find(key); - if (it == end()) return 0; - erase(it); - return 1; - } - - // Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`, - // this method returns void to reduce algorithmic complexity to O(1). The - // iterator is invalidated, so any increment should be done before calling - // erase. In order to erase while iterating across a map, use the following - // idiom (which also works for standard containers): - // - // for (auto it = m.begin(), end = m.end(); it != end;) { - // // `erase()` will invalidate `it`, so advance `it` first. - // auto copy_it = it++; - // if (<pred>) { - // m.erase(copy_it); - // } - // } - void erase(const_iterator cit) { erase(cit.inner_); } - - // This overload is necessary because otherwise erase<K>(const K&) would be - // a better match if non-const iterator is passed as an argument. - void erase(iterator it) { + // For example: + // + // std::unordered_set<ArenaString> s; + // // Makes ArenaStr even if "abc" is in the map. + // s.insert(ArenaString(&arena, "abc")); + // + // flat_hash_set<ArenaStr> s; + // // Makes ArenaStr only if "abc" is not in the map. + // s.lazy_emplace("abc", [&](const constructor& ctor) { + // ctor(&arena, "abc"); + // }); + // + // WARNING: This API is currently experimental. If there is a way to implement + // the same thing with the rest of the API, prefer that. + class constructor { + friend class raw_hash_set; + + public: + template <class... Args> + void operator()(Args&&... args) const { + assert(*slot_); + PolicyTraits::construct(alloc_, *slot_, std::forward<Args>(args)...); + *slot_ = nullptr; + } + + private: + constructor(allocator_type* a, slot_type** slot) : alloc_(a), slot_(slot) {} + + allocator_type* alloc_; + slot_type** slot_; + }; + + template <class K = key_type, class F> + iterator lazy_emplace(const key_arg<K>& key, F&& f) { + auto res = find_or_prepare_insert(key); + if (res.second) { + slot_type* slot = slots_ + res.first; + std::forward<F>(f)(constructor(&alloc_ref(), &slot)); + assert(!slot); + } + return iterator_at(res.first); + } + + // Extension API: support for heterogeneous keys. + // + // std::unordered_set<std::string> s; + // // Turns "abc" into std::string. + // s.erase("abc"); + // + // flat_hash_set<std::string> s; + // // Uses "abc" directly without copying it into std::string. + // s.erase("abc"); + template <class K = key_type> + size_type erase(const key_arg<K>& key) { + auto it = find(key); + if (it == end()) return 0; + erase(it); + return 1; + } + + // Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`, + // this method returns void to reduce algorithmic complexity to O(1). The + // iterator is invalidated, so any increment should be done before calling + // erase. In order to erase while iterating across a map, use the following + // idiom (which also works for standard containers): + // + // for (auto it = m.begin(), end = m.end(); it != end;) { + // // `erase()` will invalidate `it`, so advance `it` first. + // auto copy_it = it++; + // if (<pred>) { + // m.erase(copy_it); + // } + // } + void erase(const_iterator cit) { erase(cit.inner_); } + + // This overload is necessary because otherwise erase<K>(const K&) would be + // a better match if non-const iterator is passed as an argument. + void erase(iterator it) { AssertIsFull(it.ctrl_); - PolicyTraits::destroy(&alloc_ref(), it.slot_); - erase_meta_only(it); - } - - iterator erase(const_iterator first, const_iterator last) { - while (first != last) { - erase(first++); - } - return last.inner_; - } - - // Moves elements from `src` into `this`. - // If the element already exists in `this`, it is left unmodified in `src`. - template <typename H, typename E> - void merge(raw_hash_set<Policy, H, E, Alloc>& src) { // NOLINT - assert(this != &src); - for (auto it = src.begin(), e = src.end(); it != e;) { - auto next = std::next(it); - if (PolicyTraits::apply(InsertSlot<false>{*this, std::move(*it.slot_)}, - PolicyTraits::element(it.slot_)) - .second) { - src.erase_meta_only(it); - } - it = next; - } - } - - template <typename H, typename E> - void merge(raw_hash_set<Policy, H, E, Alloc>&& src) { - merge(src); - } - - node_type extract(const_iterator position) { + PolicyTraits::destroy(&alloc_ref(), it.slot_); + erase_meta_only(it); + } + + iterator erase(const_iterator first, const_iterator last) { + while (first != last) { + erase(first++); + } + return last.inner_; + } + + // Moves elements from `src` into `this`. + // If the element already exists in `this`, it is left unmodified in `src`. + template <typename H, typename E> + void merge(raw_hash_set<Policy, H, E, Alloc>& src) { // NOLINT + assert(this != &src); + for (auto it = src.begin(), e = src.end(); it != e;) { + auto next = std::next(it); + if (PolicyTraits::apply(InsertSlot<false>{*this, std::move(*it.slot_)}, + PolicyTraits::element(it.slot_)) + .second) { + src.erase_meta_only(it); + } + it = next; + } + } + + template <typename H, typename E> + void merge(raw_hash_set<Policy, H, E, Alloc>&& src) { + merge(src); + } + + node_type extract(const_iterator position) { AssertIsFull(position.inner_.ctrl_); - auto node = - CommonAccess::Transfer<node_type>(alloc_ref(), position.inner_.slot_); - erase_meta_only(position); - return node; - } - - template < - class K = key_type, - typename std::enable_if<!std::is_same<K, iterator>::value, int>::type = 0> - node_type extract(const key_arg<K>& key) { - auto it = find(key); - return it == end() ? node_type() : extract(const_iterator{it}); - } - - void swap(raw_hash_set& that) noexcept( - IsNoThrowSwappable<hasher>() && IsNoThrowSwappable<key_equal>() && + auto node = + CommonAccess::Transfer<node_type>(alloc_ref(), position.inner_.slot_); + erase_meta_only(position); + return node; + } + + template < + class K = key_type, + typename std::enable_if<!std::is_same<K, iterator>::value, int>::type = 0> + node_type extract(const key_arg<K>& key) { + auto it = find(key); + return it == end() ? node_type() : extract(const_iterator{it}); + } + + void swap(raw_hash_set& that) noexcept( + IsNoThrowSwappable<hasher>() && IsNoThrowSwappable<key_equal>() && IsNoThrowSwappable<allocator_type>( typename AllocTraits::propagate_on_container_swap{})) { - using std::swap; - swap(ctrl_, that.ctrl_); - swap(slots_, that.slots_); - swap(size_, that.size_); - swap(capacity_, that.capacity_); - swap(growth_left(), that.growth_left()); - swap(hash_ref(), that.hash_ref()); - swap(eq_ref(), that.eq_ref()); + using std::swap; + swap(ctrl_, that.ctrl_); + swap(slots_, that.slots_); + swap(size_, that.size_); + swap(capacity_, that.capacity_); + swap(growth_left(), that.growth_left()); + swap(hash_ref(), that.hash_ref()); + swap(eq_ref(), that.eq_ref()); swap(infoz(), that.infoz()); SwapAlloc(alloc_ref(), that.alloc_ref(), typename AllocTraits::propagate_on_container_swap{}); - } - - void rehash(size_t n) { - if (n == 0 && capacity_ == 0) return; - if (n == 0 && size_ == 0) { - destroy_slots(); + } + + void rehash(size_t n) { + if (n == 0 && capacity_ == 0) return; + if (n == 0 && size_ == 0) { + destroy_slots(); infoz().RecordStorageChanged(0, 0); infoz().RecordClearedReservation(); - return; - } + return; + } - // bitor is a faster way of doing `max` here. We will round up to the next - // power-of-2-minus-1, so bitor is good enough. - auto m = NormalizeCapacity(n | GrowthToLowerboundCapacity(size())); - // n == 0 unconditionally rehashes as per the standard. - if (n == 0 || m > capacity_) { - resize(m); + // bitor is a faster way of doing `max` here. We will round up to the next + // power-of-2-minus-1, so bitor is good enough. + auto m = NormalizeCapacity(n | GrowthToLowerboundCapacity(size())); + // n == 0 unconditionally rehashes as per the standard. + if (n == 0 || m > capacity_) { + resize(m); // This is after resize, to ensure that we have completed the allocation // and have potentially sampled the hashtable. infoz().RecordReservation(n); - } - } - + } + } + void reserve(size_t n) { if (n > size() + growth_left()) { size_t m = GrowthToLowerboundCapacity(n); @@ -1422,230 +1422,230 @@ class raw_hash_set { infoz().RecordReservation(n); } } - - // Extension API: support for heterogeneous keys. - // - // std::unordered_set<std::string> s; - // // Turns "abc" into std::string. - // s.count("abc"); - // - // ch_set<std::string> s; - // // Uses "abc" directly without copying it into std::string. - // s.count("abc"); - template <class K = key_type> - size_t count(const key_arg<K>& key) const { - return find(key) == end() ? 0 : 1; - } - - // Issues CPU prefetch instructions for the memory needed to find or insert - // a key. Like all lookup functions, this support heterogeneous keys. - // - // NOTE: This is a very low level operation and should not be used without - // specific benchmarks indicating its importance. - template <class K = key_type> - void prefetch(const key_arg<K>& key) const { - (void)key; -#if defined(__GNUC__) + + // Extension API: support for heterogeneous keys. + // + // std::unordered_set<std::string> s; + // // Turns "abc" into std::string. + // s.count("abc"); + // + // ch_set<std::string> s; + // // Uses "abc" directly without copying it into std::string. + // s.count("abc"); + template <class K = key_type> + size_t count(const key_arg<K>& key) const { + return find(key) == end() ? 0 : 1; + } + + // Issues CPU prefetch instructions for the memory needed to find or insert + // a key. Like all lookup functions, this support heterogeneous keys. + // + // NOTE: This is a very low level operation and should not be used without + // specific benchmarks indicating its importance. + template <class K = key_type> + void prefetch(const key_arg<K>& key) const { + (void)key; +#if defined(__GNUC__) prefetch_heap_block(); auto seq = probe(ctrl_, hash_ref()(key), capacity_); - __builtin_prefetch(static_cast<const void*>(ctrl_ + seq.offset())); - __builtin_prefetch(static_cast<const void*>(slots_ + seq.offset())); -#endif // __GNUC__ - } - - // The API of find() has two extensions. - // - // 1. The hash can be passed by the user. It must be equal to the hash of the - // key. - // - // 2. The type of the key argument doesn't have to be key_type. This is so - // called heterogeneous key support. - template <class K = key_type> - iterator find(const key_arg<K>& key, size_t hash) { + __builtin_prefetch(static_cast<const void*>(ctrl_ + seq.offset())); + __builtin_prefetch(static_cast<const void*>(slots_ + seq.offset())); +#endif // __GNUC__ + } + + // The API of find() has two extensions. + // + // 1. The hash can be passed by the user. It must be equal to the hash of the + // key. + // + // 2. The type of the key argument doesn't have to be key_type. This is so + // called heterogeneous key support. + template <class K = key_type> + iterator find(const key_arg<K>& key, size_t hash) { auto seq = probe(ctrl_, hash, capacity_); - while (true) { - Group g{ctrl_ + seq.offset()}; - for (int i : g.Match(H2(hash))) { - if (ABSL_PREDICT_TRUE(PolicyTraits::apply( - EqualElement<K>{key, eq_ref()}, - PolicyTraits::element(slots_ + seq.offset(i))))) - return iterator_at(seq.offset(i)); - } - if (ABSL_PREDICT_TRUE(g.MatchEmpty())) return end(); - seq.next(); + while (true) { + Group g{ctrl_ + seq.offset()}; + for (int i : g.Match(H2(hash))) { + if (ABSL_PREDICT_TRUE(PolicyTraits::apply( + EqualElement<K>{key, eq_ref()}, + PolicyTraits::element(slots_ + seq.offset(i))))) + return iterator_at(seq.offset(i)); + } + if (ABSL_PREDICT_TRUE(g.MatchEmpty())) return end(); + seq.next(); assert(seq.index() <= capacity_ && "full table!"); - } - } - template <class K = key_type> - iterator find(const key_arg<K>& key) { + } + } + template <class K = key_type> + iterator find(const key_arg<K>& key) { prefetch_heap_block(); - return find(key, hash_ref()(key)); - } - - template <class K = key_type> - const_iterator find(const key_arg<K>& key, size_t hash) const { - return const_cast<raw_hash_set*>(this)->find(key, hash); - } - template <class K = key_type> - const_iterator find(const key_arg<K>& key) const { + return find(key, hash_ref()(key)); + } + + template <class K = key_type> + const_iterator find(const key_arg<K>& key, size_t hash) const { + return const_cast<raw_hash_set*>(this)->find(key, hash); + } + template <class K = key_type> + const_iterator find(const key_arg<K>& key) const { prefetch_heap_block(); - return find(key, hash_ref()(key)); - } - - template <class K = key_type> - bool contains(const key_arg<K>& key) const { - return find(key) != end(); - } - - template <class K = key_type> - std::pair<iterator, iterator> equal_range(const key_arg<K>& key) { - auto it = find(key); - if (it != end()) return {it, std::next(it)}; - return {it, it}; - } - template <class K = key_type> - std::pair<const_iterator, const_iterator> equal_range( - const key_arg<K>& key) const { - auto it = find(key); - if (it != end()) return {it, std::next(it)}; - return {it, it}; - } - - size_t bucket_count() const { return capacity_; } - float load_factor() const { - return capacity_ ? static_cast<double>(size()) / capacity_ : 0.0; - } - float max_load_factor() const { return 1.0f; } - void max_load_factor(float) { - // Does nothing. - } - - hasher hash_function() const { return hash_ref(); } - key_equal key_eq() const { return eq_ref(); } - allocator_type get_allocator() const { return alloc_ref(); } - - friend bool operator==(const raw_hash_set& a, const raw_hash_set& b) { - if (a.size() != b.size()) return false; - const raw_hash_set* outer = &a; - const raw_hash_set* inner = &b; - if (outer->capacity() > inner->capacity()) std::swap(outer, inner); - for (const value_type& elem : *outer) - if (!inner->has_element(elem)) return false; - return true; - } - - friend bool operator!=(const raw_hash_set& a, const raw_hash_set& b) { - return !(a == b); - } - - friend void swap(raw_hash_set& a, - raw_hash_set& b) noexcept(noexcept(a.swap(b))) { - a.swap(b); - } - - private: - template <class Container, typename Enabler> - friend struct absl::container_internal::hashtable_debug_internal:: - HashtableDebugAccess; - - struct FindElement { - template <class K, class... Args> - const_iterator operator()(const K& key, Args&&...) const { - return s.find(key); - } - const raw_hash_set& s; - }; - - struct HashElement { - template <class K, class... Args> - size_t operator()(const K& key, Args&&...) const { - return h(key); - } - const hasher& h; - }; - - template <class K1> - struct EqualElement { - template <class K2, class... Args> - bool operator()(const K2& lhs, Args&&...) const { - return eq(lhs, rhs); - } - const K1& rhs; - const key_equal& eq; - }; - - struct EmplaceDecomposable { - template <class K, class... Args> - std::pair<iterator, bool> operator()(const K& key, Args&&... args) const { - auto res = s.find_or_prepare_insert(key); - if (res.second) { - s.emplace_at(res.first, std::forward<Args>(args)...); - } - return {s.iterator_at(res.first), res.second}; - } - raw_hash_set& s; - }; - - template <bool do_destroy> - struct InsertSlot { - template <class K, class... Args> - std::pair<iterator, bool> operator()(const K& key, Args&&...) && { - auto res = s.find_or_prepare_insert(key); - if (res.second) { - PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot); - } else if (do_destroy) { - PolicyTraits::destroy(&s.alloc_ref(), &slot); - } - return {s.iterator_at(res.first), res.second}; - } - raw_hash_set& s; - // Constructed slot. Either moved into place or destroyed. - slot_type&& slot; - }; - - // "erases" the object from the container, except that it doesn't actually - // destroy the object. It only updates all the metadata of the class. - // This can be used in conjunction with Policy::transfer to move the object to - // another place. - void erase_meta_only(const_iterator it) { - assert(IsFull(*it.inner_.ctrl_) && "erasing a dangling iterator"); - --size_; - const size_t index = it.inner_.ctrl_ - ctrl_; - const size_t index_before = (index - Group::kWidth) & capacity_; - const auto empty_after = Group(it.inner_.ctrl_).MatchEmpty(); - const auto empty_before = Group(ctrl_ + index_before).MatchEmpty(); - - // We count how many consecutive non empties we have to the right and to the - // left of `it`. If the sum is >= kWidth then there is at least one probe - // window that might have seen a full group. - bool was_never_full = - empty_before && empty_after && - static_cast<size_t>(empty_after.TrailingZeros() + - empty_before.LeadingZeros()) < Group::kWidth; - + return find(key, hash_ref()(key)); + } + + template <class K = key_type> + bool contains(const key_arg<K>& key) const { + return find(key) != end(); + } + + template <class K = key_type> + std::pair<iterator, iterator> equal_range(const key_arg<K>& key) { + auto it = find(key); + if (it != end()) return {it, std::next(it)}; + return {it, it}; + } + template <class K = key_type> + std::pair<const_iterator, const_iterator> equal_range( + const key_arg<K>& key) const { + auto it = find(key); + if (it != end()) return {it, std::next(it)}; + return {it, it}; + } + + size_t bucket_count() const { return capacity_; } + float load_factor() const { + return capacity_ ? static_cast<double>(size()) / capacity_ : 0.0; + } + float max_load_factor() const { return 1.0f; } + void max_load_factor(float) { + // Does nothing. + } + + hasher hash_function() const { return hash_ref(); } + key_equal key_eq() const { return eq_ref(); } + allocator_type get_allocator() const { return alloc_ref(); } + + friend bool operator==(const raw_hash_set& a, const raw_hash_set& b) { + if (a.size() != b.size()) return false; + const raw_hash_set* outer = &a; + const raw_hash_set* inner = &b; + if (outer->capacity() > inner->capacity()) std::swap(outer, inner); + for (const value_type& elem : *outer) + if (!inner->has_element(elem)) return false; + return true; + } + + friend bool operator!=(const raw_hash_set& a, const raw_hash_set& b) { + return !(a == b); + } + + friend void swap(raw_hash_set& a, + raw_hash_set& b) noexcept(noexcept(a.swap(b))) { + a.swap(b); + } + + private: + template <class Container, typename Enabler> + friend struct absl::container_internal::hashtable_debug_internal:: + HashtableDebugAccess; + + struct FindElement { + template <class K, class... Args> + const_iterator operator()(const K& key, Args&&...) const { + return s.find(key); + } + const raw_hash_set& s; + }; + + struct HashElement { + template <class K, class... Args> + size_t operator()(const K& key, Args&&...) const { + return h(key); + } + const hasher& h; + }; + + template <class K1> + struct EqualElement { + template <class K2, class... Args> + bool operator()(const K2& lhs, Args&&...) const { + return eq(lhs, rhs); + } + const K1& rhs; + const key_equal& eq; + }; + + struct EmplaceDecomposable { + template <class K, class... Args> + std::pair<iterator, bool> operator()(const K& key, Args&&... args) const { + auto res = s.find_or_prepare_insert(key); + if (res.second) { + s.emplace_at(res.first, std::forward<Args>(args)...); + } + return {s.iterator_at(res.first), res.second}; + } + raw_hash_set& s; + }; + + template <bool do_destroy> + struct InsertSlot { + template <class K, class... Args> + std::pair<iterator, bool> operator()(const K& key, Args&&...) && { + auto res = s.find_or_prepare_insert(key); + if (res.second) { + PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot); + } else if (do_destroy) { + PolicyTraits::destroy(&s.alloc_ref(), &slot); + } + return {s.iterator_at(res.first), res.second}; + } + raw_hash_set& s; + // Constructed slot. Either moved into place or destroyed. + slot_type&& slot; + }; + + // "erases" the object from the container, except that it doesn't actually + // destroy the object. It only updates all the metadata of the class. + // This can be used in conjunction with Policy::transfer to move the object to + // another place. + void erase_meta_only(const_iterator it) { + assert(IsFull(*it.inner_.ctrl_) && "erasing a dangling iterator"); + --size_; + const size_t index = it.inner_.ctrl_ - ctrl_; + const size_t index_before = (index - Group::kWidth) & capacity_; + const auto empty_after = Group(it.inner_.ctrl_).MatchEmpty(); + const auto empty_before = Group(ctrl_ + index_before).MatchEmpty(); + + // We count how many consecutive non empties we have to the right and to the + // left of `it`. If the sum is >= kWidth then there is at least one probe + // window that might have seen a full group. + bool was_never_full = + empty_before && empty_after && + static_cast<size_t>(empty_after.TrailingZeros() + + empty_before.LeadingZeros()) < Group::kWidth; + SetCtrl(index, was_never_full ? ctrl_t::kEmpty : ctrl_t::kDeleted, capacity_, ctrl_, slots_, sizeof(slot_type)); - growth_left() += was_never_full; + growth_left() += was_never_full; infoz().RecordErase(); - } - - void initialize_slots() { - assert(capacity_); - // Folks with custom allocators often make unwarranted assumptions about the - // behavior of their classes vis-a-vis trivial destructability and what - // calls they will or wont make. Avoid sampling for people with custom - // allocators to get us out of this mess. This is not a hard guarantee but - // a workaround while we plan the exact guarantee we want to provide. - // - // People are often sloppy with the exact type of their allocator (sometimes - // it has an extra const or is missing the pair, but rebinds made it work - // anyway). To avoid the ambiguity, we work off SlotAlloc which we have - // bound more carefully. - if (std::is_same<SlotAlloc, std::allocator<slot_type>>::value && - slots_ == nullptr) { + } + + void initialize_slots() { + assert(capacity_); + // Folks with custom allocators often make unwarranted assumptions about the + // behavior of their classes vis-a-vis trivial destructability and what + // calls they will or wont make. Avoid sampling for people with custom + // allocators to get us out of this mess. This is not a hard guarantee but + // a workaround while we plan the exact guarantee we want to provide. + // + // People are often sloppy with the exact type of their allocator (sometimes + // it has an extra const or is missing the pair, but rebinds made it work + // anyway). To avoid the ambiguity, we work off SlotAlloc which we have + // bound more carefully. + if (std::is_same<SlotAlloc, std::allocator<slot_type>>::value && + slots_ == nullptr) { infoz() = Sample(sizeof(slot_type)); - } - + } + char* mem = static_cast<char*>(Allocate<alignof(slot_type)>( &alloc_ref(), AllocSize(capacity_, sizeof(slot_type), alignof(slot_type)))); @@ -1653,133 +1653,133 @@ class raw_hash_set { slots_ = reinterpret_cast<slot_type*>( mem + SlotOffset(capacity_, alignof(slot_type))); ResetCtrl(capacity_, ctrl_, slots_, sizeof(slot_type)); - reset_growth_left(); + reset_growth_left(); infoz().RecordStorageChanged(size_, capacity_); - } - - void destroy_slots() { - if (!capacity_) return; - for (size_t i = 0; i != capacity_; ++i) { - if (IsFull(ctrl_[i])) { - PolicyTraits::destroy(&alloc_ref(), slots_ + i); - } - } - - // Unpoison before returning the memory to the allocator. - SanitizerUnpoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_); + } + + void destroy_slots() { + if (!capacity_) return; + for (size_t i = 0; i != capacity_; ++i) { + if (IsFull(ctrl_[i])) { + PolicyTraits::destroy(&alloc_ref(), slots_ + i); + } + } + + // Unpoison before returning the memory to the allocator. + SanitizerUnpoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_); Deallocate<alignof(slot_type)>( &alloc_ref(), ctrl_, AllocSize(capacity_, sizeof(slot_type), alignof(slot_type))); - ctrl_ = EmptyGroup(); - slots_ = nullptr; - size_ = 0; - capacity_ = 0; - growth_left() = 0; - } - - void resize(size_t new_capacity) { - assert(IsValidCapacity(new_capacity)); - auto* old_ctrl = ctrl_; - auto* old_slots = slots_; - const size_t old_capacity = capacity_; - capacity_ = new_capacity; - initialize_slots(); - - size_t total_probe_length = 0; - for (size_t i = 0; i != old_capacity; ++i) { - if (IsFull(old_ctrl[i])) { - size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, - PolicyTraits::element(old_slots + i)); + ctrl_ = EmptyGroup(); + slots_ = nullptr; + size_ = 0; + capacity_ = 0; + growth_left() = 0; + } + + void resize(size_t new_capacity) { + assert(IsValidCapacity(new_capacity)); + auto* old_ctrl = ctrl_; + auto* old_slots = slots_; + const size_t old_capacity = capacity_; + capacity_ = new_capacity; + initialize_slots(); + + size_t total_probe_length = 0; + for (size_t i = 0; i != old_capacity; ++i) { + if (IsFull(old_ctrl[i])) { + size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, + PolicyTraits::element(old_slots + i)); auto target = find_first_non_full(ctrl_, hash, capacity_); - size_t new_i = target.offset; - total_probe_length += target.probe_length; + size_t new_i = target.offset; + total_probe_length += target.probe_length; SetCtrl(new_i, H2(hash), capacity_, ctrl_, slots_, sizeof(slot_type)); - PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, old_slots + i); - } - } - if (old_capacity) { - SanitizerUnpoisonMemoryRegion(old_slots, - sizeof(slot_type) * old_capacity); + PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, old_slots + i); + } + } + if (old_capacity) { + SanitizerUnpoisonMemoryRegion(old_slots, + sizeof(slot_type) * old_capacity); Deallocate<alignof(slot_type)>( &alloc_ref(), old_ctrl, AllocSize(old_capacity, sizeof(slot_type), alignof(slot_type))); - } + } infoz().RecordRehash(total_probe_length); - } - - void drop_deletes_without_resize() ABSL_ATTRIBUTE_NOINLINE { - assert(IsValidCapacity(capacity_)); + } + + void drop_deletes_without_resize() ABSL_ATTRIBUTE_NOINLINE { + assert(IsValidCapacity(capacity_)); assert(!is_small(capacity_)); - // Algorithm: - // - mark all DELETED slots as EMPTY - // - mark all FULL slots as DELETED - // - for each slot marked as DELETED - // hash = Hash(element) - // target = find_first_non_full(hash) - // if target is in the same group - // mark slot as FULL - // else if target is EMPTY - // transfer element to target - // mark slot as EMPTY - // mark target as FULL - // else if target is DELETED - // swap current element with target element - // mark target as FULL - // repeat procedure for current slot with moved from element (target) - ConvertDeletedToEmptyAndFullToDeleted(ctrl_, capacity_); + // Algorithm: + // - mark all DELETED slots as EMPTY + // - mark all FULL slots as DELETED + // - for each slot marked as DELETED + // hash = Hash(element) + // target = find_first_non_full(hash) + // if target is in the same group + // mark slot as FULL + // else if target is EMPTY + // transfer element to target + // mark slot as EMPTY + // mark target as FULL + // else if target is DELETED + // swap current element with target element + // mark target as FULL + // repeat procedure for current slot with moved from element (target) + ConvertDeletedToEmptyAndFullToDeleted(ctrl_, capacity_); alignas(slot_type) unsigned char raw[sizeof(slot_type)]; - size_t total_probe_length = 0; - slot_type* slot = reinterpret_cast<slot_type*>(&raw); - for (size_t i = 0; i != capacity_; ++i) { - if (!IsDeleted(ctrl_[i])) continue; + size_t total_probe_length = 0; + slot_type* slot = reinterpret_cast<slot_type*>(&raw); + for (size_t i = 0; i != capacity_; ++i) { + if (!IsDeleted(ctrl_[i])) continue; const size_t hash = PolicyTraits::apply( HashElement{hash_ref()}, PolicyTraits::element(slots_ + i)); const FindInfo target = find_first_non_full(ctrl_, hash, capacity_); const size_t new_i = target.offset; - total_probe_length += target.probe_length; - - // Verify if the old and new i fall within the same group wrt the hash. - // If they do, we don't need to move the object as it falls already in the - // best probe we can. + total_probe_length += target.probe_length; + + // Verify if the old and new i fall within the same group wrt the hash. + // If they do, we don't need to move the object as it falls already in the + // best probe we can. const size_t probe_offset = probe(ctrl_, hash, capacity_).offset(); const auto probe_index = [probe_offset, this](size_t pos) { return ((pos - probe_offset) & capacity_) / Group::kWidth; - }; - - // Element doesn't move. - if (ABSL_PREDICT_TRUE(probe_index(new_i) == probe_index(i))) { + }; + + // Element doesn't move. + if (ABSL_PREDICT_TRUE(probe_index(new_i) == probe_index(i))) { SetCtrl(i, H2(hash), capacity_, ctrl_, slots_, sizeof(slot_type)); - continue; - } - if (IsEmpty(ctrl_[new_i])) { - // Transfer element to the empty spot. + continue; + } + if (IsEmpty(ctrl_[new_i])) { + // Transfer element to the empty spot. // SetCtrl poisons/unpoisons the slots so we have to call it at the - // right time. + // right time. SetCtrl(new_i, H2(hash), capacity_, ctrl_, slots_, sizeof(slot_type)); - PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slots_ + i); + PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slots_ + i); SetCtrl(i, ctrl_t::kEmpty, capacity_, ctrl_, slots_, sizeof(slot_type)); - } else { - assert(IsDeleted(ctrl_[new_i])); + } else { + assert(IsDeleted(ctrl_[new_i])); SetCtrl(new_i, H2(hash), capacity_, ctrl_, slots_, sizeof(slot_type)); - // Until we are done rehashing, DELETED marks previously FULL slots. - // Swap i and new_i elements. - PolicyTraits::transfer(&alloc_ref(), slot, slots_ + i); - PolicyTraits::transfer(&alloc_ref(), slots_ + i, slots_ + new_i); - PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slot); - --i; // repeat - } - } - reset_growth_left(); + // Until we are done rehashing, DELETED marks previously FULL slots. + // Swap i and new_i elements. + PolicyTraits::transfer(&alloc_ref(), slot, slots_ + i); + PolicyTraits::transfer(&alloc_ref(), slots_ + i, slots_ + new_i); + PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slot); + --i; // repeat + } + } + reset_growth_left(); infoz().RecordRehash(total_probe_length); - } - - void rehash_and_grow_if_necessary() { - if (capacity_ == 0) { - resize(1); + } + + void rehash_and_grow_if_necessary() { + if (capacity_ == 0) { + resize(1); } else if (capacity_ > Group::kWidth && // Do these calcuations in 64-bit to avoid overflow. size() * uint64_t{32} <= capacity_ * uint64_t{25}) { - // Squash DELETED without growing if there is enough capacity. + // Squash DELETED without growing if there is enough capacity. // // Rehash in place if the current size is <= 25/32 of capacity_. // Rationale for such a high factor: 1) drop_deletes_without_resize() is @@ -1820,108 +1820,108 @@ class raw_hash_set { // 762 | 149836 0.37 13 | 148559 0.74 190 // 807 | 149736 0.39 14 | 151107 0.39 14 // 852 | 150204 0.42 15 | 151019 0.42 15 - drop_deletes_without_resize(); - } else { - // Otherwise grow the container. - resize(capacity_ * 2 + 1); - } - } - - bool has_element(const value_type& elem) const { - size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, elem); + drop_deletes_without_resize(); + } else { + // Otherwise grow the container. + resize(capacity_ * 2 + 1); + } + } + + bool has_element(const value_type& elem) const { + size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, elem); auto seq = probe(ctrl_, hash, capacity_); - while (true) { - Group g{ctrl_ + seq.offset()}; - for (int i : g.Match(H2(hash))) { - if (ABSL_PREDICT_TRUE(PolicyTraits::element(slots_ + seq.offset(i)) == - elem)) - return true; - } - if (ABSL_PREDICT_TRUE(g.MatchEmpty())) return false; - seq.next(); + while (true) { + Group g{ctrl_ + seq.offset()}; + for (int i : g.Match(H2(hash))) { + if (ABSL_PREDICT_TRUE(PolicyTraits::element(slots_ + seq.offset(i)) == + elem)) + return true; + } + if (ABSL_PREDICT_TRUE(g.MatchEmpty())) return false; + seq.next(); assert(seq.index() <= capacity_ && "full table!"); - } - return false; - } - - // TODO(alkis): Optimize this assuming *this and that don't overlap. - raw_hash_set& move_assign(raw_hash_set&& that, std::true_type) { - raw_hash_set tmp(std::move(that)); - swap(tmp); - return *this; - } - raw_hash_set& move_assign(raw_hash_set&& that, std::false_type) { - raw_hash_set tmp(std::move(that), alloc_ref()); - swap(tmp); - return *this; - } - - protected: - template <class K> - std::pair<size_t, bool> find_or_prepare_insert(const K& key) { + } + return false; + } + + // TODO(alkis): Optimize this assuming *this and that don't overlap. + raw_hash_set& move_assign(raw_hash_set&& that, std::true_type) { + raw_hash_set tmp(std::move(that)); + swap(tmp); + return *this; + } + raw_hash_set& move_assign(raw_hash_set&& that, std::false_type) { + raw_hash_set tmp(std::move(that), alloc_ref()); + swap(tmp); + return *this; + } + + protected: + template <class K> + std::pair<size_t, bool> find_or_prepare_insert(const K& key) { prefetch_heap_block(); - auto hash = hash_ref()(key); + auto hash = hash_ref()(key); auto seq = probe(ctrl_, hash, capacity_); - while (true) { - Group g{ctrl_ + seq.offset()}; - for (int i : g.Match(H2(hash))) { - if (ABSL_PREDICT_TRUE(PolicyTraits::apply( - EqualElement<K>{key, eq_ref()}, - PolicyTraits::element(slots_ + seq.offset(i))))) - return {seq.offset(i), false}; - } - if (ABSL_PREDICT_TRUE(g.MatchEmpty())) break; - seq.next(); + while (true) { + Group g{ctrl_ + seq.offset()}; + for (int i : g.Match(H2(hash))) { + if (ABSL_PREDICT_TRUE(PolicyTraits::apply( + EqualElement<K>{key, eq_ref()}, + PolicyTraits::element(slots_ + seq.offset(i))))) + return {seq.offset(i), false}; + } + if (ABSL_PREDICT_TRUE(g.MatchEmpty())) break; + seq.next(); assert(seq.index() <= capacity_ && "full table!"); - } - return {prepare_insert(hash), true}; - } - - size_t prepare_insert(size_t hash) ABSL_ATTRIBUTE_NOINLINE { + } + return {prepare_insert(hash), true}; + } + + size_t prepare_insert(size_t hash) ABSL_ATTRIBUTE_NOINLINE { auto target = find_first_non_full(ctrl_, hash, capacity_); - if (ABSL_PREDICT_FALSE(growth_left() == 0 && - !IsDeleted(ctrl_[target.offset]))) { - rehash_and_grow_if_necessary(); + if (ABSL_PREDICT_FALSE(growth_left() == 0 && + !IsDeleted(ctrl_[target.offset]))) { + rehash_and_grow_if_necessary(); target = find_first_non_full(ctrl_, hash, capacity_); - } - ++size_; - growth_left() -= IsEmpty(ctrl_[target.offset]); + } + ++size_; + growth_left() -= IsEmpty(ctrl_[target.offset]); SetCtrl(target.offset, H2(hash), capacity_, ctrl_, slots_, sizeof(slot_type)); infoz().RecordInsert(hash, target.probe_length); - return target.offset; - } - - // Constructs the value in the space pointed by the iterator. This only works - // after an unsuccessful find_or_prepare_insert() and before any other - // modifications happen in the raw_hash_set. - // - // PRECONDITION: i is an index returned from find_or_prepare_insert(k), where - // k is the key decomposed from `forward<Args>(args)...`, and the bool - // returned by find_or_prepare_insert(k) was true. - // POSTCONDITION: *m.iterator_at(i) == value_type(forward<Args>(args)...). - template <class... Args> - void emplace_at(size_t i, Args&&... args) { - PolicyTraits::construct(&alloc_ref(), slots_ + i, - std::forward<Args>(args)...); - - assert(PolicyTraits::apply(FindElement{*this}, *iterator_at(i)) == - iterator_at(i) && - "constructed value does not match the lookup key"); - } - - iterator iterator_at(size_t i) { return {ctrl_ + i, slots_ + i}; } - const_iterator iterator_at(size_t i) const { return {ctrl_ + i, slots_ + i}; } - - private: - friend struct RawHashSetTestOnlyAccess; - - void reset_growth_left() { - growth_left() = CapacityToGrowth(capacity()) - size_; - } - + return target.offset; + } + + // Constructs the value in the space pointed by the iterator. This only works + // after an unsuccessful find_or_prepare_insert() and before any other + // modifications happen in the raw_hash_set. + // + // PRECONDITION: i is an index returned from find_or_prepare_insert(k), where + // k is the key decomposed from `forward<Args>(args)...`, and the bool + // returned by find_or_prepare_insert(k) was true. + // POSTCONDITION: *m.iterator_at(i) == value_type(forward<Args>(args)...). + template <class... Args> + void emplace_at(size_t i, Args&&... args) { + PolicyTraits::construct(&alloc_ref(), slots_ + i, + std::forward<Args>(args)...); + + assert(PolicyTraits::apply(FindElement{*this}, *iterator_at(i)) == + iterator_at(i) && + "constructed value does not match the lookup key"); + } + + iterator iterator_at(size_t i) { return {ctrl_ + i, slots_ + i}; } + const_iterator iterator_at(size_t i) const { return {ctrl_ + i, slots_ + i}; } + + private: + friend struct RawHashSetTestOnlyAccess; + + void reset_growth_left() { + growth_left() = CapacityToGrowth(capacity()) - size_; + } + size_t& growth_left() { return settings_.template get<0>(); } - + void prefetch_heap_block() const { // Prefetch the heap-allocated memory region to resolve potential TLB // misses. This is intended to overlap with execution of calculating the @@ -1929,33 +1929,33 @@ class raw_hash_set { #if defined(__GNUC__) __builtin_prefetch(static_cast<const void*>(ctrl_), 0, 1); #endif // __GNUC__ - } - + } + HashtablezInfoHandle& infoz() { return settings_.template get<1>(); } - + hasher& hash_ref() { return settings_.template get<2>(); } const hasher& hash_ref() const { return settings_.template get<2>(); } key_equal& eq_ref() { return settings_.template get<3>(); } const key_equal& eq_ref() const { return settings_.template get<3>(); } allocator_type& alloc_ref() { return settings_.template get<4>(); } - const allocator_type& alloc_ref() const { + const allocator_type& alloc_ref() const { return settings_.template get<4>(); - } - - // TODO(alkis): Investigate removing some of these fields: - // - ctrl/slots can be derived from each other - // - size can be moved into the slot array + } + + // TODO(alkis): Investigate removing some of these fields: + // - ctrl/slots can be derived from each other + // - size can be moved into the slot array ctrl_t* ctrl_ = EmptyGroup(); // [(capacity + 1 + NumClonedBytes()) * ctrl_t] slot_type* slots_ = nullptr; // [capacity * slot_type] size_t size_ = 0; // number of full slots size_t capacity_ = 0; // total number of slots absl::container_internal::CompressedTuple<size_t /* growth_left */, HashtablezInfoHandle, hasher, - key_equal, allocator_type> + key_equal, allocator_type> settings_{0, HashtablezInfoHandle{}, hasher{}, key_equal{}, allocator_type{}}; -}; - +}; + // Erases all elements that satisfy the predicate `pred` from the container `c`. template <typename P, typename H, typename E, typename A, typename Predicate> void EraseIf(Predicate& pred, raw_hash_set<P, H, E, A>* c) { @@ -1968,67 +1968,67 @@ void EraseIf(Predicate& pred, raw_hash_set<P, H, E, A>* c) { } } -namespace hashtable_debug_internal { -template <typename Set> -struct HashtableDebugAccess<Set, absl::void_t<typename Set::raw_hash_set>> { - using Traits = typename Set::PolicyTraits; - using Slot = typename Traits::slot_type; - - static size_t GetNumProbes(const Set& set, - const typename Set::key_type& key) { - size_t num_probes = 0; - size_t hash = set.hash_ref()(key); +namespace hashtable_debug_internal { +template <typename Set> +struct HashtableDebugAccess<Set, absl::void_t<typename Set::raw_hash_set>> { + using Traits = typename Set::PolicyTraits; + using Slot = typename Traits::slot_type; + + static size_t GetNumProbes(const Set& set, + const typename Set::key_type& key) { + size_t num_probes = 0; + size_t hash = set.hash_ref()(key); auto seq = probe(set.ctrl_, hash, set.capacity_); - while (true) { - container_internal::Group g{set.ctrl_ + seq.offset()}; - for (int i : g.Match(container_internal::H2(hash))) { - if (Traits::apply( - typename Set::template EqualElement<typename Set::key_type>{ - key, set.eq_ref()}, - Traits::element(set.slots_ + seq.offset(i)))) - return num_probes; - ++num_probes; - } - if (g.MatchEmpty()) return num_probes; - seq.next(); - ++num_probes; - } - } - - static size_t AllocatedByteSize(const Set& c) { - size_t capacity = c.capacity_; - if (capacity == 0) return 0; + while (true) { + container_internal::Group g{set.ctrl_ + seq.offset()}; + for (int i : g.Match(container_internal::H2(hash))) { + if (Traits::apply( + typename Set::template EqualElement<typename Set::key_type>{ + key, set.eq_ref()}, + Traits::element(set.slots_ + seq.offset(i)))) + return num_probes; + ++num_probes; + } + if (g.MatchEmpty()) return num_probes; + seq.next(); + ++num_probes; + } + } + + static size_t AllocatedByteSize(const Set& c) { + size_t capacity = c.capacity_; + if (capacity == 0) return 0; size_t m = AllocSize(capacity, sizeof(Slot), alignof(Slot)); - - size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr)); - if (per_slot != ~size_t{}) { - m += per_slot * c.size(); - } else { - for (size_t i = 0; i != capacity; ++i) { - if (container_internal::IsFull(c.ctrl_[i])) { - m += Traits::space_used(c.slots_ + i); - } - } - } - return m; - } - - static size_t LowerBoundAllocatedByteSize(size_t size) { - size_t capacity = GrowthToLowerboundCapacity(size); - if (capacity == 0) return 0; + + size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr)); + if (per_slot != ~size_t{}) { + m += per_slot * c.size(); + } else { + for (size_t i = 0; i != capacity; ++i) { + if (container_internal::IsFull(c.ctrl_[i])) { + m += Traits::space_used(c.slots_ + i); + } + } + } + return m; + } + + static size_t LowerBoundAllocatedByteSize(size_t size) { + size_t capacity = GrowthToLowerboundCapacity(size); + if (capacity == 0) return 0; size_t m = AllocSize(NormalizeCapacity(capacity), sizeof(Slot), alignof(Slot)); - size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr)); - if (per_slot != ~size_t{}) { - m += per_slot * size; - } - return m; - } -}; - -} // namespace hashtable_debug_internal -} // namespace container_internal + size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr)); + if (per_slot != ~size_t{}) { + m += per_slot * size; + } + return m; + } +}; + +} // namespace hashtable_debug_internal +} // namespace container_internal ABSL_NAMESPACE_END -} // namespace absl - -#endif // ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ |