diff options
author | thegeorg <thegeorg@yandex-team.com> | 2024-01-25 20:29:07 +0300 |
---|---|---|
committer | thegeorg <thegeorg@yandex-team.com> | 2024-01-25 20:51:18 +0300 |
commit | 24abb4e0b50dd362e8cf30a682d5212252936b09 (patch) | |
tree | c5356c59cfe5480daca33b63b1742680a48586e8 /contrib | |
parent | b65dd88d2d36688300317c22b9c14ed9dcdeb37d (diff) | |
download | ydb-24abb4e0b50dd362e8cf30a682d5212252936b09.tar.gz |
Update contrib/restricted/abseil-cpp to 20240116.0
Diffstat (limited to 'contrib')
158 files changed, 6712 insertions, 5227 deletions
diff --git a/contrib/restricted/abseil-cpp/absl/algorithm/algorithm.h b/contrib/restricted/abseil-cpp/absl/algorithm/algorithm.h index e9b4733872..59aeed7d26 100644 --- a/contrib/restricted/abseil-cpp/absl/algorithm/algorithm.h +++ b/contrib/restricted/abseil-cpp/absl/algorithm/algorithm.h @@ -31,92 +31,17 @@ namespace absl { ABSL_NAMESPACE_BEGIN -namespace algorithm_internal { - -// Performs comparisons with operator==, similar to C++14's `std::equal_to<>`. -struct EqualTo { - template <typename T, typename U> - bool operator()(const T& a, const U& b) const { - return a == b; - } -}; - -template <typename InputIter1, typename InputIter2, typename Pred> -bool EqualImpl(InputIter1 first1, InputIter1 last1, InputIter2 first2, - InputIter2 last2, Pred pred, std::input_iterator_tag, - std::input_iterator_tag) { - while (true) { - if (first1 == last1) return first2 == last2; - if (first2 == last2) return false; - if (!pred(*first1, *first2)) return false; - ++first1; - ++first2; - } -} - -template <typename InputIter1, typename InputIter2, typename Pred> -bool EqualImpl(InputIter1 first1, InputIter1 last1, InputIter2 first2, - InputIter2 last2, Pred&& pred, std::random_access_iterator_tag, - std::random_access_iterator_tag) { - return (last1 - first1 == last2 - first2) && - std::equal(first1, last1, first2, std::forward<Pred>(pred)); -} - -// When we are using our own internal predicate that just applies operator==, we -// forward to the non-predicate form of std::equal. This enables an optimization -// in libstdc++ that can result in std::memcmp being used for integer types. -template <typename InputIter1, typename InputIter2> -bool EqualImpl(InputIter1 first1, InputIter1 last1, InputIter2 first2, - InputIter2 last2, algorithm_internal::EqualTo /* unused */, - std::random_access_iterator_tag, - std::random_access_iterator_tag) { - return (last1 - first1 == last2 - first2) && - std::equal(first1, last1, first2); -} - -template <typename It> -It RotateImpl(It first, It middle, It last, std::true_type) { - return std::rotate(first, middle, last); -} - -template <typename It> -It RotateImpl(It first, It middle, It last, std::false_type) { - std::rotate(first, middle, last); - return std::next(first, std::distance(middle, last)); -} - -} // namespace algorithm_internal - // equal() +// rotate() // -// Compares the equality of two ranges specified by pairs of iterators, using -// the given predicate, returning true iff for each corresponding iterator i1 -// and i2 in the first and second range respectively, pred(*i1, *i2) == true -// -// This comparison takes at most min(`last1` - `first1`, `last2` - `first2`) -// invocations of the predicate. Additionally, if InputIter1 and InputIter2 are -// both random-access iterators, and `last1` - `first1` != `last2` - `first2`, -// then the predicate is never invoked and the function returns false. +// Historical note: Abseil once provided implementations of these algorithms +// prior to their adoption in C++14. New code should prefer to use the std +// variants. // -// This is a C++11-compatible implementation of C++14 `std::equal`. See -// https://en.cppreference.com/w/cpp/algorithm/equal for more information. -template <typename InputIter1, typename InputIter2, typename Pred> -bool equal(InputIter1 first1, InputIter1 last1, InputIter2 first2, - InputIter2 last2, Pred&& pred) { - return algorithm_internal::EqualImpl( - first1, last1, first2, last2, std::forward<Pred>(pred), - typename std::iterator_traits<InputIter1>::iterator_category{}, - typename std::iterator_traits<InputIter2>::iterator_category{}); -} - -// Overload of equal() that performs comparison of two ranges specified by pairs -// of iterators using operator==. -template <typename InputIter1, typename InputIter2> -bool equal(InputIter1 first1, InputIter1 last1, InputIter2 first2, - InputIter2 last2) { - return absl::equal(first1, last1, first2, last2, - algorithm_internal::EqualTo{}); -} +// See the documentation for the STL <algorithm> header for more information: +// https://en.cppreference.com/w/cpp/header/algorithm +using std::equal; +using std::rotate; // linear_search() // @@ -133,26 +58,6 @@ bool linear_search(InputIterator first, InputIterator last, return std::find(first, last, value) != last; } -// rotate() -// -// Performs a left rotation on a range of elements (`first`, `last`) such that -// `middle` is now the first element. `rotate()` returns an iterator pointing to -// the first element before rotation. This function is exactly the same as -// `std::rotate`, but fixes a bug in gcc -// <= 4.9 where `std::rotate` returns `void` instead of an iterator. -// -// The complexity of this algorithm is the same as that of `std::rotate`, but if -// `ForwardIterator` is not a random-access iterator, then `absl::rotate` -// performs an additional pass over the range to construct the return value. -template <typename ForwardIterator> -ForwardIterator rotate(ForwardIterator first, ForwardIterator middle, - ForwardIterator last) { - return algorithm_internal::RotateImpl( - first, middle, last, - std::is_same<decltype(std::rotate(first, middle, last)), - ForwardIterator>()); -} - ABSL_NAMESPACE_END } // namespace absl diff --git a/contrib/restricted/abseil-cpp/absl/algorithm/container.h b/contrib/restricted/abseil-cpp/absl/algorithm/container.h index 679e02676c..c7bafae147 100644 --- a/contrib/restricted/abseil-cpp/absl/algorithm/container.h +++ b/contrib/restricted/abseil-cpp/absl/algorithm/container.h @@ -52,6 +52,7 @@ #include "absl/algorithm/algorithm.h" #include "absl/base/macros.h" +#include "absl/base/nullability.h" #include "absl/meta/type_traits.h" namespace absl { @@ -116,18 +117,6 @@ template <class Key, class Hash, class KeyEqual, class Allocator> struct IsUnorderedContainer<std::unordered_set<Key, Hash, KeyEqual, Allocator>> : std::true_type {}; -// container_algorithm_internal::c_size. It is meant for internal use only. - -template <class C> -auto c_size(C& c) -> decltype(c.size()) { - return c.size(); -} - -template <class T, std::size_t N> -constexpr std::size_t c_size(T (&)[N]) { - return N; -} - } // namespace container_algorithm_internal // PUBLIC API @@ -348,20 +337,10 @@ container_algorithm_internal::ContainerDifferenceType<const C> c_count_if( template <typename C1, typename C2> container_algorithm_internal::ContainerIterPairType<C1, C2> c_mismatch(C1& c1, C2& c2) { - auto first1 = container_algorithm_internal::c_begin(c1); - auto last1 = container_algorithm_internal::c_end(c1); - auto first2 = container_algorithm_internal::c_begin(c2); - auto last2 = container_algorithm_internal::c_end(c2); - - for (; first1 != last1 && first2 != last2; ++first1, (void)++first2) { - // Negates equality because Cpp17EqualityComparable doesn't require clients - // to overload both `operator==` and `operator!=`. - if (!(*first1 == *first2)) { - break; - } - } - - return std::make_pair(first1, first2); + return std::mismatch(container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2), + container_algorithm_internal::c_end(c2)); } // Overload of c_mismatch() for using a predicate evaluation other than `==` as @@ -370,56 +349,33 @@ container_algorithm_internal::ContainerIterPairType<C1, C2> c_mismatch(C1& c1, template <typename C1, typename C2, typename BinaryPredicate> container_algorithm_internal::ContainerIterPairType<C1, C2> c_mismatch( C1& c1, C2& c2, BinaryPredicate pred) { - auto first1 = container_algorithm_internal::c_begin(c1); - auto last1 = container_algorithm_internal::c_end(c1); - auto first2 = container_algorithm_internal::c_begin(c2); - auto last2 = container_algorithm_internal::c_end(c2); - - for (; first1 != last1 && first2 != last2; ++first1, (void)++first2) { - if (!pred(*first1, *first2)) { - break; - } - } - - return std::make_pair(first1, first2); + return std::mismatch(container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2), + container_algorithm_internal::c_end(c2), pred); } // c_equal() // // Container-based version of the <algorithm> `std::equal()` function to // test whether two containers are equal. -// -// NOTE: the semantics of c_equal() are slightly different than those of -// equal(): while the latter iterates over the second container only up to the -// size of the first container, c_equal() also checks whether the container -// sizes are equal. This better matches expectations about c_equal() based on -// its signature. -// -// Example: -// vector v1 = <1, 2, 3>; -// vector v2 = <1, 2, 3, 4>; -// equal(std::begin(v1), std::end(v1), std::begin(v2)) returns true -// c_equal(v1, v2) returns false - template <typename C1, typename C2> bool c_equal(const C1& c1, const C2& c2) { - return ((container_algorithm_internal::c_size(c1) == - container_algorithm_internal::c_size(c2)) && - std::equal(container_algorithm_internal::c_begin(c1), - container_algorithm_internal::c_end(c1), - container_algorithm_internal::c_begin(c2))); + return std::equal(container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2), + container_algorithm_internal::c_end(c2)); } // Overload of c_equal() for using a predicate evaluation other than `==` as // the function's test condition. template <typename C1, typename C2, typename BinaryPredicate> bool c_equal(const C1& c1, const C2& c2, BinaryPredicate&& pred) { - return ((container_algorithm_internal::c_size(c1) == - container_algorithm_internal::c_size(c2)) && - std::equal(container_algorithm_internal::c_begin(c1), - container_algorithm_internal::c_end(c1), - container_algorithm_internal::c_begin(c2), - std::forward<BinaryPredicate>(pred))); + return std::equal(container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2), + container_algorithm_internal::c_end(c2), + std::forward<BinaryPredicate>(pred)); } // c_is_permutation() @@ -428,20 +384,20 @@ bool c_equal(const C1& c1, const C2& c2, BinaryPredicate&& pred) { // to test whether a container is a permutation of another. template <typename C1, typename C2> bool c_is_permutation(const C1& c1, const C2& c2) { - using std::begin; - using std::end; - return c1.size() == c2.size() && - std::is_permutation(begin(c1), end(c1), begin(c2)); + return std::is_permutation(container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2), + container_algorithm_internal::c_end(c2)); } // Overload of c_is_permutation() for using a predicate evaluation other than // `==` as the function's test condition. template <typename C1, typename C2, typename BinaryPredicate> bool c_is_permutation(const C1& c1, const C2& c2, BinaryPredicate&& pred) { - using std::begin; - using std::end; - return c1.size() == c2.size() && - std::is_permutation(begin(c1), end(c1), begin(c2), + return std::is_permutation(container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2), + container_algorithm_internal::c_end(c2), std::forward<BinaryPredicate>(pred)); } @@ -818,6 +774,36 @@ void c_shuffle(RandomAccessContainer& c, UniformRandomBitGenerator&& gen) { std::forward<UniformRandomBitGenerator>(gen)); } +// c_sample() +// +// Container-based version of the <algorithm> `std::sample()` function to +// randomly sample elements from the container without replacement using a +// `gen()` uniform random number generator and write them to an iterator range. +template <typename C, typename OutputIterator, typename Distance, + typename UniformRandomBitGenerator> +OutputIterator c_sample(const C& c, OutputIterator result, Distance n, + UniformRandomBitGenerator&& gen) { +#if defined(__cpp_lib_sample) && __cpp_lib_sample >= 201603L + return std::sample(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), result, n, + std::forward<UniformRandomBitGenerator>(gen)); +#else + // Fall back to a stable selection-sampling implementation. + auto first = container_algorithm_internal::c_begin(c); + Distance unsampled_elements = c_distance(c); + n = (std::min)(n, unsampled_elements); + for (; n != 0; ++first) { + Distance r = + std::uniform_int_distribution<Distance>(0, --unsampled_elements)(gen); + if (r < n) { + *result++ = *first; + --n; + } + } + return result; +#endif +} + //------------------------------------------------------------------------------ // <algorithm> Partition functions //------------------------------------------------------------------------------ @@ -1657,7 +1643,7 @@ bool c_prev_permutation(C& c, LessThan&& comp) { // // Container-based version of the <numeric> `std::iota()` function // to compute successive values of `value`, as if incremented with `++value` -// after each element is written. and write them to the container. +// after each element is written, and write them to the container. template <typename Sequence, typename T> void c_iota(Sequence& sequence, const T& value) { std::iota(container_algorithm_internal::c_begin(sequence), diff --git a/contrib/restricted/abseil-cpp/absl/algorithm/ya.make b/contrib/restricted/abseil-cpp/absl/algorithm/ya.make index 3b66e3e400..3f9af85fe6 100644 --- a/contrib/restricted/abseil-cpp/absl/algorithm/ya.make +++ b/contrib/restricted/abseil-cpp/absl/algorithm/ya.make @@ -6,9 +6,9 @@ LICENSE(Apache-2.0) LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(20230802.1) +VERSION(20240116.0) -ORIGINAL_SOURCE(https://github.com/abseil/abseil-cpp/archive/20230802.1.tar.gz) +ORIGINAL_SOURCE(https://github.com/abseil/abseil-cpp/archive/20240116.0.tar.gz) NO_RUNTIME() diff --git a/contrib/restricted/abseil-cpp/absl/base/attributes.h b/contrib/restricted/abseil-cpp/absl/base/attributes.h index a7f279a025..d4f67a1295 100644 --- a/contrib/restricted/abseil-cpp/absl/base/attributes.h +++ b/contrib/restricted/abseil-cpp/absl/base/attributes.h @@ -687,7 +687,7 @@ // When deprecating Abseil code, it is sometimes necessary to turn off the // warning within Abseil, until the deprecated code is actually removed. The -// deprecated code can be surrounded with these directives to acheive that +// deprecated code can be surrounded with these directives to achieve that // result. // // class ABSL_DEPRECATED("Use Bar instead") Foo; @@ -747,9 +747,52 @@ #define ABSL_CONST_INIT #endif -// These annotations are not available yet due to fear of breaking code. -#define ABSL_ATTRIBUTE_PURE_FUNCTION -#define ABSL_ATTRIBUTE_CONST_FUNCTION +// ABSL_ATTRIBUTE_PURE_FUNCTION +// +// ABSL_ATTRIBUTE_PURE_FUNCTION is used to annotate declarations of "pure" +// functions. A function is pure if its return value is only a function of its +// arguments. The pure attribute prohibits a function from modifying the state +// of the program that is observable by means other than inspecting the +// function's return value. Declaring such functions with the pure attribute +// allows the compiler to avoid emitting some calls in repeated invocations of +// the function with the same argument values. +// +// Example: +// +// ABSL_ATTRIBUTE_PURE_FUNCTION std::string FormatTime(Time t); +#if ABSL_HAVE_CPP_ATTRIBUTE(gnu::pure) +#define ABSL_ATTRIBUTE_PURE_FUNCTION [[gnu::pure]] +#elif ABSL_HAVE_ATTRIBUTE(pure) +#define ABSL_ATTRIBUTE_PURE_FUNCTION __attribute__((pure)) +#else +// If the attribute isn't defined, we'll fallback to ABSL_MUST_USE_RESULT since +// pure functions are useless if its return is ignored. +#define ABSL_ATTRIBUTE_PURE_FUNCTION ABSL_MUST_USE_RESULT +#endif + +// ABSL_ATTRIBUTE_CONST_FUNCTION +// +// ABSL_ATTRIBUTE_CONST_FUNCTION is used to annotate declarations of "const" +// functions. A const function is similar to a pure function, with one +// exception: Pure functions may return value that depend on a non-volatile +// object that isn't provided as a function argument, while the const function +// is guaranteed to return the same result given the same arguments. +// +// Example: +// +// ABSL_ATTRIBUTE_CONST_FUNCTION int64_t ToInt64Milliseconds(Duration d); +#if defined(_MSC_VER) && !defined(__clang__) +// Put the MSVC case first since MSVC seems to parse const as a C++ keyword. +#define ABSL_ATTRIBUTE_CONST_FUNCTION ABSL_ATTRIBUTE_PURE_FUNCTION +#elif ABSL_HAVE_CPP_ATTRIBUTE(gnu::const) +#define ABSL_ATTRIBUTE_CONST_FUNCTION [[gnu::const]] +#elif ABSL_HAVE_ATTRIBUTE(const) +#define ABSL_ATTRIBUTE_CONST_FUNCTION __attribute__((const)) +#else +// Since const functions are more restrictive pure function, we'll fallback to a +// pure function if the const attribute is not handled. +#define ABSL_ATTRIBUTE_CONST_FUNCTION ABSL_ATTRIBUTE_PURE_FUNCTION +#endif // ABSL_ATTRIBUTE_LIFETIME_BOUND indicates that a resource owned by a function // parameter or implicit object parameter is retained by the return value of the @@ -800,15 +843,11 @@ // See also the upstream documentation: // https://clang.llvm.org/docs/AttributeReference.html#trivial-abi // -#if ABSL_HAVE_CPP_ATTRIBUTE(clang::trivial_abi) -#define ABSL_ATTRIBUTE_TRIVIAL_ABI [[clang::trivial_abi]] -#define ABSL_HAVE_ATTRIBUTE_TRIVIAL_ABI 1 -#elif ABSL_HAVE_ATTRIBUTE(trivial_abi) -#define ABSL_ATTRIBUTE_TRIVIAL_ABI __attribute__((trivial_abi)) -#define ABSL_HAVE_ATTRIBUTE_TRIVIAL_ABI 1 -#else +// b/321691395 - This is currently disabled in open-source builds since +// compiler support differs. If system libraries compiled with GCC are mixed +// with libraries compiled with Clang, types will have different ideas about +// their ABI, leading to hard to debug crashes. #define ABSL_ATTRIBUTE_TRIVIAL_ABI -#endif // ABSL_ATTRIBUTE_NO_UNIQUE_ADDRESS // diff --git a/contrib/restricted/abseil-cpp/absl/base/call_once.h b/contrib/restricted/abseil-cpp/absl/base/call_once.h index 08436bac8a..7b0e69ccd5 100644 --- a/contrib/restricted/abseil-cpp/absl/base/call_once.h +++ b/contrib/restricted/abseil-cpp/absl/base/call_once.h @@ -37,6 +37,7 @@ #include "absl/base/internal/scheduling_mode.h" #include "absl/base/internal/spinlock_wait.h" #include "absl/base/macros.h" +#include "absl/base/nullability.h" #include "absl/base/optimization.h" #include "absl/base/port.h" @@ -46,7 +47,8 @@ ABSL_NAMESPACE_BEGIN class once_flag; namespace base_internal { -std::atomic<uint32_t>* ControlWord(absl::once_flag* flag); +absl::Nonnull<std::atomic<uint32_t>*> ControlWord( + absl::Nonnull<absl::once_flag*> flag); } // namespace base_internal // call_once() @@ -89,7 +91,8 @@ class once_flag { once_flag& operator=(const once_flag&) = delete; private: - friend std::atomic<uint32_t>* base_internal::ControlWord(once_flag* flag); + friend absl::Nonnull<std::atomic<uint32_t>*> base_internal::ControlWord( + absl::Nonnull<once_flag*> flag); std::atomic<uint32_t> control_; }; @@ -103,7 +106,8 @@ namespace base_internal { // Like call_once, but uses KERNEL_ONLY scheduling. Intended to be used to // initialize entities used by the scheduler implementation. template <typename Callable, typename... Args> -void LowLevelCallOnce(absl::once_flag* flag, Callable&& fn, Args&&... args); +void LowLevelCallOnce(absl::Nonnull<absl::once_flag*> flag, Callable&& fn, + Args&&... args); // Disables scheduling while on stack when scheduling mode is non-cooperative. // No effect for cooperative scheduling modes. @@ -143,10 +147,10 @@ enum { }; template <typename Callable, typename... Args> -ABSL_ATTRIBUTE_NOINLINE -void CallOnceImpl(std::atomic<uint32_t>* control, - base_internal::SchedulingMode scheduling_mode, Callable&& fn, - Args&&... args) { +ABSL_ATTRIBUTE_NOINLINE void CallOnceImpl( + absl::Nonnull<std::atomic<uint32_t>*> control, + base_internal::SchedulingMode scheduling_mode, Callable&& fn, + Args&&... args) { #ifndef NDEBUG { uint32_t old_control = control->load(std::memory_order_relaxed); @@ -185,12 +189,14 @@ void CallOnceImpl(std::atomic<uint32_t>* control, } // else *control is already kOnceDone } -inline std::atomic<uint32_t>* ControlWord(once_flag* flag) { +inline absl::Nonnull<std::atomic<uint32_t>*> ControlWord( + absl::Nonnull<once_flag*> flag) { return &flag->control_; } template <typename Callable, typename... Args> -void LowLevelCallOnce(absl::once_flag* flag, Callable&& fn, Args&&... args) { +void LowLevelCallOnce(absl::Nonnull<absl::once_flag*> flag, Callable&& fn, + Args&&... args) { std::atomic<uint32_t>* once = base_internal::ControlWord(flag); uint32_t s = once->load(std::memory_order_acquire); if (ABSL_PREDICT_FALSE(s != base_internal::kOnceDone)) { diff --git a/contrib/restricted/abseil-cpp/absl/base/casts.h b/contrib/restricted/abseil-cpp/absl/base/casts.h index d19588852f..e0b11bbe49 100644 --- a/contrib/restricted/abseil-cpp/absl/base/casts.h +++ b/contrib/restricted/abseil-cpp/absl/base/casts.h @@ -90,7 +90,7 @@ ABSL_NAMESPACE_BEGIN // // Such implicit cast chaining may be useful within template logic. template <typename To> -constexpr To implicit_cast(typename absl::internal::identity_t<To> to) { +constexpr To implicit_cast(typename absl::internal::type_identity_t<To> to) { return to; } diff --git a/contrib/restricted/abseil-cpp/absl/base/config.h b/contrib/restricted/abseil-cpp/absl/base/config.h index 334e2767ac..8bd9a6d586 100644 --- a/contrib/restricted/abseil-cpp/absl/base/config.h +++ b/contrib/restricted/abseil-cpp/absl/base/config.h @@ -75,6 +75,12 @@ #define ABSL_INTERNAL_CPLUSPLUS_LANG __cplusplus #endif +#if defined(ABSL_INTERNAL_CPLUSPLUS_LANG) && \ + ABSL_INTERNAL_CPLUSPLUS_LANG >= 202002L +// Include library feature test macros. +#include <version> +#endif + #if defined(__APPLE__) // Included for TARGET_OS_IPHONE, __IPHONE_OS_VERSION_MIN_REQUIRED, // __IPHONE_8_0. @@ -111,8 +117,8 @@ // // LTS releases can be obtained from // https://github.com/abseil/abseil-cpp/releases. -#define ABSL_LTS_RELEASE_VERSION 20230802 -#define ABSL_LTS_RELEASE_PATCH_LEVEL 1 +#define ABSL_LTS_RELEASE_VERSION 20240116 +#define ABSL_LTS_RELEASE_PATCH_LEVEL 0 // Helper macro to convert a CPP variable to a string literal. #define ABSL_INTERNAL_DO_TOKEN_STR(x) #x @@ -332,8 +338,8 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || #ifdef ABSL_HAVE_INTRINSIC_INT128 #error ABSL_HAVE_INTRINSIC_INT128 cannot be directly set #elif defined(__SIZEOF_INT128__) -#if (defined(__clang__) && !defined(_WIN32)) || \ - (defined(__CUDACC__) && __CUDACC_VER_MAJOR__ >= 9) || \ +#if (defined(__clang__) && !defined(_WIN32)) || \ + (defined(__CUDACC__) && __CUDACC_VER_MAJOR__ >= 9) || \ (defined(__GNUC__) && !defined(__clang__) && !defined(__CUDACC__)) #define ABSL_HAVE_INTRINSIC_INT128 1 #elif defined(__CUDACC__) @@ -395,7 +401,7 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || // Windows _WIN32 // NaCL __native_client__ // AsmJS __asmjs__ -// WebAssembly __wasm__ +// WebAssembly (Emscripten) __EMSCRIPTEN__ // Fuchsia __Fuchsia__ // // Note that since Android defines both __ANDROID__ and __linux__, one @@ -407,11 +413,11 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || // POSIX.1-2001. #ifdef ABSL_HAVE_MMAP #error ABSL_HAVE_MMAP cannot be directly set -#elif defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || \ - defined(_AIX) || defined(__ros__) || defined(__native_client__) || \ - defined(__asmjs__) || defined(__wasm__) || defined(__Fuchsia__) || \ - defined(__sun) || defined(__ASYLO__) || defined(__myriad2__) || \ - defined(__HAIKU__) || defined(__OpenBSD__) || defined(__NetBSD__) || \ +#elif defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || \ + defined(_AIX) || defined(__ros__) || defined(__native_client__) || \ + defined(__asmjs__) || defined(__EMSCRIPTEN__) || defined(__Fuchsia__) || \ + defined(__sun) || defined(__ASYLO__) || defined(__myriad2__) || \ + defined(__HAIKU__) || defined(__OpenBSD__) || defined(__NetBSD__) || \ defined(__QNX__) || defined(__VXWORKS__) || defined(__hexagon__) #define ABSL_HAVE_MMAP 1 #endif @@ -484,6 +490,8 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || // https://sourceforge.net/p/mingw-w64/mingw-w64/ci/master/tree/mingw-w64-crt/misc/alarm.c #elif defined(__EMSCRIPTEN__) // emscripten doesn't support signals +#elif defined(__wasi__) +// WASI doesn't support signals #elif defined(__Fuchsia__) // Signals don't exist on fuchsia. #elif defined(__native_client__) @@ -536,14 +544,14 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || // and // https://github.com/llvm/llvm-project/commit/0bc451e7e137c4ccadcd3377250874f641ca514a // The second has the actually correct versions, thus, is what we copy here. -#if defined(__APPLE__) && \ - ((defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \ - __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101300) || \ - (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && \ - __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 120000) || \ - (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && \ - __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 50000) || \ - (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && \ +#if defined(__APPLE__) && \ + ((defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \ + __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101300) || \ + (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && \ + __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 120000) || \ + (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && \ + __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 50000) || \ + (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && \ __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 120000)) #define ABSL_INTERNAL_APPLE_CXX17_TYPES_UNAVAILABLE 1 #else @@ -555,7 +563,7 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || // Checks whether C++17 std::any is available. #ifdef ABSL_HAVE_STD_ANY #error "ABSL_HAVE_STD_ANY cannot be directly set." -#elif defined(__cpp_lib_any) +#elif defined(__cpp_lib_any) && __cpp_lib_any >= 201606L #define ABSL_HAVE_STD_ANY 1 #elif defined(ABSL_INTERNAL_CPLUSPLUS_LANG) && \ ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L && \ @@ -568,9 +576,9 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || // Checks whether C++17 std::optional is available. #ifdef ABSL_HAVE_STD_OPTIONAL #error "ABSL_HAVE_STD_OPTIONAL cannot be directly set." -#elif defined(__cpp_lib_optional) +#elif defined(__cpp_lib_optional) && __cpp_lib_optional >= 202106L #define ABSL_HAVE_STD_OPTIONAL 1 -#elif defined(ABSL_INTERNAL_CPLUSPLUS_LANG) && \ +#elif defined(ABSL_INTERNAL_CPLUSPLUS_LANG) && \ ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L && \ !ABSL_INTERNAL_APPLE_CXX17_TYPES_UNAVAILABLE #define ABSL_HAVE_STD_OPTIONAL 1 @@ -581,7 +589,7 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || // Checks whether C++17 std::variant is available. #ifdef ABSL_HAVE_STD_VARIANT #error "ABSL_HAVE_STD_VARIANT cannot be directly set." -#elif defined(__cpp_lib_variant) +#elif defined(__cpp_lib_variant) && __cpp_lib_variant >= 201606L #define ABSL_HAVE_STD_VARIANT 1 #elif defined(ABSL_INTERNAL_CPLUSPLUS_LANG) && \ ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L && \ @@ -594,13 +602,29 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || // Checks whether C++17 std::string_view is available. #ifdef ABSL_HAVE_STD_STRING_VIEW #error "ABSL_HAVE_STD_STRING_VIEW cannot be directly set." -#elif defined(__cpp_lib_string_view) +#elif defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L #define ABSL_HAVE_STD_STRING_VIEW 1 #elif defined(ABSL_INTERNAL_CPLUSPLUS_LANG) && \ ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L #define ABSL_HAVE_STD_STRING_VIEW 1 #endif +// ABSL_HAVE_STD_ORDERING +// +// Checks whether C++20 std::{partial,weak,strong}_ordering are available. +// +// __cpp_lib_three_way_comparison is missing on libc++ +// (https://github.com/llvm/llvm-project/issues/73953) so treat it as defined +// when building in C++20 mode. +#ifdef ABSL_HAVE_STD_ORDERING +#error "ABSL_HAVE_STD_ORDERING cannot be directly set." +#elif (defined(__cpp_lib_three_way_comparison) && \ + __cpp_lib_three_way_comparison >= 201907L) || \ + (defined(ABSL_INTERNAL_CPLUSPLUS_LANG) && \ + ABSL_INTERNAL_CPLUSPLUS_LANG >= 202002L) +#define ABSL_HAVE_STD_ORDERING 1 +#endif + // ABSL_USES_STD_ANY // // Indicates whether absl::any is an alias for std::any. @@ -663,6 +687,22 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || #error options.h is misconfigured. #endif +// ABSL_USES_STD_ORDERING +// +// Indicates whether absl::{partial,weak,strong}_ordering are aliases for the +// std:: ordering types. +#if !defined(ABSL_OPTION_USE_STD_ORDERING) +#error options.h is misconfigured. +#elif ABSL_OPTION_USE_STD_ORDERING == 0 || \ + (ABSL_OPTION_USE_STD_ORDERING == 2 && !defined(ABSL_HAVE_STD_ORDERING)) +#undef ABSL_USES_STD_ORDERING +#elif ABSL_OPTION_USE_STD_ORDERING == 1 || \ + (ABSL_OPTION_USE_STD_ORDERING == 2 && defined(ABSL_HAVE_STD_ORDERING)) +#define ABSL_USES_STD_ORDERING 1 +#else +#error options.h is misconfigured. +#endif + // In debug mode, MSVC 2017's std::variant throws a EXCEPTION_ACCESS_VIOLATION // SEH exception from emplace for variant<SomeStruct> when constructing the // struct can throw. This defeats some of variant_test and @@ -845,11 +885,30 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || // RTTI support. #ifdef ABSL_INTERNAL_HAS_RTTI #error ABSL_INTERNAL_HAS_RTTI cannot be directly set -#elif (defined(__GNUC__) && defined(__GXX_RTTI)) || \ - (defined(_MSC_VER) && defined(_CPPRTTI)) || \ - (!defined(__GNUC__) && !defined(_MSC_VER)) +#elif ABSL_HAVE_FEATURE(cxx_rtti) +#define ABSL_INTERNAL_HAS_RTTI 1 +#elif defined(__GNUC__) && defined(__GXX_RTTI) #define ABSL_INTERNAL_HAS_RTTI 1 -#endif // !defined(__GNUC__) || defined(__GXX_RTTI) +#elif defined(_MSC_VER) && defined(_CPPRTTI) +#define ABSL_INTERNAL_HAS_RTTI 1 +#elif !defined(__GNUC__) && !defined(_MSC_VER) +// Unknown compiler, default to RTTI +#define ABSL_INTERNAL_HAS_RTTI 1 +#endif + +// `ABSL_INTERNAL_HAS_CXA_DEMANGLE` determines whether `abi::__cxa_demangle` is +// available. +#ifdef ABSL_INTERNAL_HAS_CXA_DEMANGLE +#error ABSL_INTERNAL_HAS_CXA_DEMANGLE cannot be directly set +#elif defined(OS_ANDROID) && (defined(__i386__) || defined(__x86_64__)) +#define ABSL_INTERNAL_HAS_CXA_DEMANGLE 0 +#elif defined(__GNUC__) && defined(__GNUC_MINOR__) && \ + (__GNUC__ >= 4 || (__GNUC__ >= 3 && __GNUC_MINOR__ >= 4)) && \ + !defined(__mips__) +#define ABSL_INTERNAL_HAS_CXA_DEMANGLE 1 +#elif defined(__clang__) && !defined(_MSC_VER) +#define ABSL_INTERNAL_HAS_CXA_DEMANGLE 1 +#endif // ABSL_INTERNAL_HAVE_SSE is used for compile-time detection of SSE support. // See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html for an overview of @@ -936,8 +995,8 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || #if __EMSCRIPTEN_tiny__ >= 1000 #error __EMSCRIPTEN_tiny__ is too big to fit in ABSL_INTERNAL_EMSCRIPTEN_VERSION #endif -#define ABSL_INTERNAL_EMSCRIPTEN_VERSION \ - ((__EMSCRIPTEN_major__)*1000000 + (__EMSCRIPTEN_minor__)*1000 + \ +#define ABSL_INTERNAL_EMSCRIPTEN_VERSION \ + ((__EMSCRIPTEN_major__) * 1000000 + (__EMSCRIPTEN_minor__) * 1000 + \ (__EMSCRIPTEN_tiny__)) #endif #endif diff --git a/contrib/restricted/abseil-cpp/absl/base/internal/endian.h b/contrib/restricted/abseil-cpp/absl/base/internal/endian.h index 50747d75ec..943f3d97e7 100644 --- a/contrib/restricted/abseil-cpp/absl/base/internal/endian.h +++ b/contrib/restricted/abseil-cpp/absl/base/internal/endian.h @@ -22,6 +22,7 @@ #include "absl/base/casts.h" #include "absl/base/config.h" #include "absl/base/internal/unaligned_access.h" +#include "absl/base/nullability.h" #include "absl/base/port.h" namespace absl { @@ -160,27 +161,27 @@ inline int64_t ToHost(int64_t x) { } // Functions to do unaligned loads and stores in little-endian order. -inline uint16_t Load16(const void *p) { +inline uint16_t Load16(absl::Nonnull<const void *> p) { return ToHost16(ABSL_INTERNAL_UNALIGNED_LOAD16(p)); } -inline void Store16(void *p, uint16_t v) { +inline void Store16(absl::Nonnull<void *> p, uint16_t v) { ABSL_INTERNAL_UNALIGNED_STORE16(p, FromHost16(v)); } -inline uint32_t Load32(const void *p) { +inline uint32_t Load32(absl::Nonnull<const void *> p) { return ToHost32(ABSL_INTERNAL_UNALIGNED_LOAD32(p)); } -inline void Store32(void *p, uint32_t v) { +inline void Store32(absl::Nonnull<void *> p, uint32_t v) { ABSL_INTERNAL_UNALIGNED_STORE32(p, FromHost32(v)); } -inline uint64_t Load64(const void *p) { +inline uint64_t Load64(absl::Nonnull<const void *> p) { return ToHost64(ABSL_INTERNAL_UNALIGNED_LOAD64(p)); } -inline void Store64(void *p, uint64_t v) { +inline void Store64(absl::Nonnull<void *> p, uint64_t v) { ABSL_INTERNAL_UNALIGNED_STORE64(p, FromHost64(v)); } @@ -250,27 +251,27 @@ inline int64_t ToHost(int64_t x) { } // Functions to do unaligned loads and stores in big-endian order. -inline uint16_t Load16(const void *p) { +inline uint16_t Load16(absl::Nonnull<const void *> p) { return ToHost16(ABSL_INTERNAL_UNALIGNED_LOAD16(p)); } -inline void Store16(void *p, uint16_t v) { +inline void Store16(absl::Nonnull<void *> p, uint16_t v) { ABSL_INTERNAL_UNALIGNED_STORE16(p, FromHost16(v)); } -inline uint32_t Load32(const void *p) { +inline uint32_t Load32(absl::Nonnull<const void *> p) { return ToHost32(ABSL_INTERNAL_UNALIGNED_LOAD32(p)); } -inline void Store32(void *p, uint32_t v) { +inline void Store32(absl::Nonnull<void *>p, uint32_t v) { ABSL_INTERNAL_UNALIGNED_STORE32(p, FromHost32(v)); } -inline uint64_t Load64(const void *p) { +inline uint64_t Load64(absl::Nonnull<const void *> p) { return ToHost64(ABSL_INTERNAL_UNALIGNED_LOAD64(p)); } -inline void Store64(void *p, uint64_t v) { +inline void Store64(absl::Nonnull<void *> p, uint64_t v) { ABSL_INTERNAL_UNALIGNED_STORE64(p, FromHost64(v)); } diff --git a/contrib/restricted/abseil-cpp/absl/base/internal/identity.h b/contrib/restricted/abseil-cpp/absl/base/internal/identity.h index a3154ed7bc..365207b7e7 100644 --- a/contrib/restricted/abseil-cpp/absl/base/internal/identity.h +++ b/contrib/restricted/abseil-cpp/absl/base/internal/identity.h @@ -22,13 +22,15 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace internal { +// This is a back-fill of C++20's `std::type_identity`. template <typename T> -struct identity { +struct type_identity { typedef T type; }; +// This is a back-fill of C++20's `std::type_identity_t`. template <typename T> -using identity_t = typename identity<T>::type; +using type_identity_t = typename type_identity<T>::type; } // namespace internal ABSL_NAMESPACE_END diff --git a/contrib/restricted/abseil-cpp/absl/base/internal/inline_variable.h b/contrib/restricted/abseil-cpp/absl/base/internal/inline_variable.h index df933faff5..09daf0f5b7 100644 --- a/contrib/restricted/abseil-cpp/absl/base/internal/inline_variable.h +++ b/contrib/restricted/abseil-cpp/absl/base/internal/inline_variable.h @@ -63,12 +63,12 @@ // Bug: https://bugs.llvm.org/show_bug.cgi?id=35862 // // Note: -// identity_t is used here so that the const and name are in the +// type_identity_t is used here so that the const and name are in the // appropriate place for pointer types, reference types, function pointer // types, etc.. #if defined(__clang__) #define ABSL_INTERNAL_EXTERN_DECL(type, name) \ - extern const ::absl::internal::identity_t<type> name; + extern const ::absl::internal::type_identity_t<type> name; #else // Otherwise, just define the macro to do nothing. #define ABSL_INTERNAL_EXTERN_DECL(type, name) #endif // defined(__clang__) @@ -76,30 +76,31 @@ // See above comment at top of file for details. #define ABSL_INTERNAL_INLINE_CONSTEXPR(type, name, init) \ ABSL_INTERNAL_EXTERN_DECL(type, name) \ - inline constexpr ::absl::internal::identity_t<type> name = init + inline constexpr ::absl::internal::type_identity_t<type> name = init #else // See above comment at top of file for details. // // Note: -// identity_t is used here so that the const and name are in the +// type_identity_t is used here so that the const and name are in the // appropriate place for pointer types, reference types, function pointer // types, etc.. -#define ABSL_INTERNAL_INLINE_CONSTEXPR(var_type, name, init) \ - template <class /*AbslInternalDummy*/ = void> \ - struct AbslInternalInlineVariableHolder##name { \ - static constexpr ::absl::internal::identity_t<var_type> kInstance = init; \ - }; \ - \ - template <class AbslInternalDummy> \ - constexpr ::absl::internal::identity_t<var_type> \ - AbslInternalInlineVariableHolder##name<AbslInternalDummy>::kInstance; \ - \ - static constexpr const ::absl::internal::identity_t<var_type>& \ - name = /* NOLINT */ \ - AbslInternalInlineVariableHolder##name<>::kInstance; \ - static_assert(sizeof(void (*)(decltype(name))) != 0, \ +#define ABSL_INTERNAL_INLINE_CONSTEXPR(var_type, name, init) \ + template <class /*AbslInternalDummy*/ = void> \ + struct AbslInternalInlineVariableHolder##name { \ + static constexpr ::absl::internal::type_identity_t<var_type> kInstance = \ + init; \ + }; \ + \ + template <class AbslInternalDummy> \ + constexpr ::absl::internal::type_identity_t<var_type> \ + AbslInternalInlineVariableHolder##name<AbslInternalDummy>::kInstance; \ + \ + static constexpr const ::absl::internal::type_identity_t<var_type>& \ + name = /* NOLINT */ \ + AbslInternalInlineVariableHolder##name<>::kInstance; \ + static_assert(sizeof(void (*)(decltype(name))) != 0, \ "Silence unused variable warnings.") #endif // __cpp_inline_variables diff --git a/contrib/restricted/abseil-cpp/absl/base/internal/low_level_alloc.cc b/contrib/restricted/abseil-cpp/absl/base/internal/low_level_alloc.cc index 6d2cfeac3c..a563f7b9f9 100644 --- a/contrib/restricted/abseil-cpp/absl/base/internal/low_level_alloc.cc +++ b/contrib/restricted/abseil-cpp/absl/base/internal/low_level_alloc.cc @@ -329,7 +329,7 @@ size_t GetPageSize() { SYSTEM_INFO system_info; GetSystemInfo(&system_info); return std::max(system_info.dwPageSize, system_info.dwAllocationGranularity); -#elif defined(__wasm__) || defined(__asmjs__) +#elif defined(__wasm__) || defined(__asmjs__) || defined(__hexagon__) return getpagesize(); #else return static_cast<size_t>(sysconf(_SC_PAGESIZE)); diff --git a/contrib/restricted/abseil-cpp/absl/base/internal/nullability_impl.h b/contrib/restricted/abseil-cpp/absl/base/internal/nullability_impl.h new file mode 100644 index 0000000000..36e1b33d65 --- /dev/null +++ b/contrib/restricted/abseil-cpp/absl/base/internal/nullability_impl.h @@ -0,0 +1,106 @@ +// Copyright 2023 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_INTERNAL_NULLABILITY_IMPL_H_ +#define ABSL_BASE_INTERNAL_NULLABILITY_IMPL_H_ + +#include <memory> +#include <type_traits> + +#include "absl/base/attributes.h" +#include "absl/meta/type_traits.h" + +namespace absl { + +namespace nullability_internal { + +// `IsNullabilityCompatible` checks whether its first argument is a class +// explicitly tagged as supporting nullability annotations. The tag is the type +// declaration `absl_nullability_compatible`. +template <typename, typename = void> +struct IsNullabilityCompatible : std::false_type {}; + +template <typename T> +struct IsNullabilityCompatible< + T, absl::void_t<typename T::absl_nullability_compatible>> : std::true_type { +}; + +template <typename T> +constexpr bool IsSupportedType = IsNullabilityCompatible<T>::value; + +template <typename T> +constexpr bool IsSupportedType<T*> = true; + +template <typename T, typename U> +constexpr bool IsSupportedType<T U::*> = true; + +template <typename T, typename... Deleter> +constexpr bool IsSupportedType<std::unique_ptr<T, Deleter...>> = true; + +template <typename T> +constexpr bool IsSupportedType<std::shared_ptr<T>> = true; + +template <typename T> +struct EnableNullable { + static_assert(nullability_internal::IsSupportedType<std::remove_cv_t<T>>, + "Template argument must be a raw or supported smart pointer " + "type. See absl/base/nullability.h."); + using type = T; +}; + +template <typename T> +struct EnableNonnull { + static_assert(nullability_internal::IsSupportedType<std::remove_cv_t<T>>, + "Template argument must be a raw or supported smart pointer " + "type. See absl/base/nullability.h."); + using type = T; +}; + +template <typename T> +struct EnableNullabilityUnknown { + static_assert(nullability_internal::IsSupportedType<std::remove_cv_t<T>>, + "Template argument must be a raw or supported smart pointer " + "type. See absl/base/nullability.h."); + using type = T; +}; + +// Note: we do not apply Clang nullability attributes (e.g. _Nullable). These +// only support raw pointers, and conditionally enabling them only for raw +// pointers inhibits template arg deduction. Ideally, they would support all +// pointer-like types. +template <typename T, typename = typename EnableNullable<T>::type> +using NullableImpl +#if ABSL_HAVE_CPP_ATTRIBUTE(clang::annotate) + [[clang::annotate("Nullable")]] +#endif + = T; + +template <typename T, typename = typename EnableNonnull<T>::type> +using NonnullImpl +#if ABSL_HAVE_CPP_ATTRIBUTE(clang::annotate) + [[clang::annotate("Nonnull")]] +#endif + = T; + +template <typename T, typename = typename EnableNullabilityUnknown<T>::type> +using NullabilityUnknownImpl +#if ABSL_HAVE_CPP_ATTRIBUTE(clang::annotate) + [[clang::annotate("Nullability_Unspecified")]] +#endif + = T; + +} // namespace nullability_internal +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_NULLABILITY_IMPL_H_ diff --git a/contrib/restricted/abseil-cpp/absl/base/internal/raw_logging.cc b/contrib/restricted/abseil-cpp/absl/base/internal/raw_logging.cc index a1ca876d4e..6f68002a3c 100644 --- a/contrib/restricted/abseil-cpp/absl/base/internal/raw_logging.cc +++ b/contrib/restricted/abseil-cpp/absl/base/internal/raw_logging.cc @@ -42,8 +42,9 @@ // This preprocessor token is also defined in raw_io.cc. If you need to copy // this, consider moving both to config.h instead. #if defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || \ - defined(__Fuchsia__) || defined(__native_client__) || \ - defined(__OpenBSD__) || defined(__EMSCRIPTEN__) || defined(__ASYLO__) + defined(__hexagon__) || defined(__Fuchsia__) || \ + defined(__native_client__) || defined(__OpenBSD__) || \ + defined(__EMSCRIPTEN__) || defined(__ASYLO__) #include <unistd.h> @@ -56,8 +57,7 @@ // ABSL_HAVE_SYSCALL_WRITE is defined when the platform provides the syscall // syscall(SYS_write, /*int*/ fd, /*char* */ buf, /*size_t*/ len); // for low level operations that want to avoid libc. -#if (defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__)) && \ - !defined(__ANDROID__) +#if (defined(__linux__) || defined(__FreeBSD__)) && !defined(__ANDROID__) #include <sys/syscall.h> #define ABSL_HAVE_SYSCALL_WRITE 1 #define ABSL_LOW_LEVEL_WRITE_SUPPORTED 1 @@ -93,8 +93,7 @@ constexpr char kTruncated[] = " ... (message truncated)\n"; bool VADoRawLog(char** buf, int* size, const char* format, va_list ap) ABSL_PRINTF_ATTRIBUTE(3, 0); bool VADoRawLog(char** buf, int* size, const char* format, va_list ap) { - if (*size < 0) - return false; + if (*size < 0) return false; int n = vsnprintf(*buf, static_cast<size_t>(*size), format, ap); bool result = true; if (n < 0 || n > *size) { @@ -122,8 +121,7 @@ constexpr int kLogBufSize = 3000; bool DoRawLog(char** buf, int* size, const char* format, ...) ABSL_PRINTF_ATTRIBUTE(3, 4); bool DoRawLog(char** buf, int* size, const char* format, ...) { - if (*size < 0) - return false; + if (*size < 0) return false; va_list ap; va_start(ap, format); int n = vsnprintf(*buf, static_cast<size_t>(*size), format, ap); @@ -242,8 +240,8 @@ void AsyncSignalSafeWriteError(const char* s, size_t len) { _write(/* stderr */ 2, s, static_cast<unsigned>(len)); #else // stderr logging unsupported on this platform - (void) s; - (void) len; + (void)s; + (void)len; #endif } @@ -258,7 +256,7 @@ void RawLog(absl::LogSeverity severity, const char* file, int line, bool RawLoggingFullySupported() { #ifdef ABSL_LOW_LEVEL_WRITE_SUPPORTED return true; -#else // !ABSL_LOW_LEVEL_WRITE_SUPPORTED +#else // !ABSL_LOW_LEVEL_WRITE_SUPPORTED return false; #endif // !ABSL_LOW_LEVEL_WRITE_SUPPORTED } diff --git a/contrib/restricted/abseil-cpp/absl/base/internal/raw_logging.h b/contrib/restricted/abseil-cpp/absl/base/internal/raw_logging.h index b79550b2b7..d7cfbc576b 100644 --- a/contrib/restricted/abseil-cpp/absl/base/internal/raw_logging.h +++ b/contrib/restricted/abseil-cpp/absl/base/internal/raw_logging.h @@ -108,6 +108,7 @@ #define ABSL_RAW_LOG_INTERNAL_WARNING ::absl::LogSeverity::kWarning #define ABSL_RAW_LOG_INTERNAL_ERROR ::absl::LogSeverity::kError #define ABSL_RAW_LOG_INTERNAL_FATAL ::absl::LogSeverity::kFatal +#define ABSL_RAW_LOG_INTERNAL_DFATAL ::absl::kLogDebugFatal #define ABSL_RAW_LOG_INTERNAL_LEVEL(severity) \ ::absl::NormalizeLogSeverity(severity) @@ -115,6 +116,7 @@ #define ABSL_RAW_LOG_INTERNAL_MAYBE_UNREACHABLE_WARNING #define ABSL_RAW_LOG_INTERNAL_MAYBE_UNREACHABLE_ERROR #define ABSL_RAW_LOG_INTERNAL_MAYBE_UNREACHABLE_FATAL ABSL_UNREACHABLE() +#define ABSL_RAW_LOG_INTERNAL_MAYBE_UNREACHABLE_DFATAL #define ABSL_RAW_LOG_INTERNAL_MAYBE_UNREACHABLE_LEVEL(severity) namespace absl { diff --git a/contrib/restricted/abseil-cpp/absl/base/internal/spinlock.h b/contrib/restricted/abseil-cpp/absl/base/internal/spinlock.h index 09ba5824b1..2929cd6f5c 100644 --- a/contrib/restricted/abseil-cpp/absl/base/internal/spinlock.h +++ b/contrib/restricted/abseil-cpp/absl/base/internal/spinlock.h @@ -19,10 +19,10 @@ // - for use by Abseil internal code that Mutex itself depends on // - for async signal safety (see below) -// SpinLock is async signal safe. If a spinlock is used within a signal -// handler, all code that acquires the lock must ensure that the signal cannot -// arrive while they are holding the lock. Typically, this is done by blocking -// the signal. +// SpinLock with a base_internal::SchedulingMode::SCHEDULE_KERNEL_ONLY is async +// signal safe. If a spinlock is used within a signal handler, all code that +// acquires the lock must ensure that the signal cannot arrive while they are +// holding the lock. Typically, this is done by blocking the signal. // // Threads waiting on a SpinLock may be woken in an arbitrary order. @@ -41,6 +41,14 @@ #include "absl/base/internal/tsan_mutex_interface.h" #include "absl/base/thread_annotations.h" +namespace tcmalloc { +namespace tcmalloc_internal { + +class AllocationGuardSpinLockHolder; + +} // namespace tcmalloc_internal +} // namespace tcmalloc + namespace absl { ABSL_NAMESPACE_BEGIN namespace base_internal { @@ -137,6 +145,7 @@ class ABSL_LOCKABLE SpinLock { // Provide access to protected method above. Use for testing only. friend struct SpinLockTest; + friend class tcmalloc::tcmalloc_internal::AllocationGuardSpinLockHolder; private: // lockword_ is used to store the following: @@ -171,6 +180,10 @@ class ABSL_LOCKABLE SpinLock { return scheduling_mode == base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL; } + bool IsCooperative() const { + return lockword_.load(std::memory_order_relaxed) & kSpinLockCooperative; + } + uint32_t TryLockInternal(uint32_t lock_value, uint32_t wait_cycles); void SlowLock() ABSL_ATTRIBUTE_COLD; void SlowUnlock(uint32_t lock_value) ABSL_ATTRIBUTE_COLD; diff --git a/contrib/restricted/abseil-cpp/absl/base/internal/thread_annotations.h b/contrib/restricted/abseil-cpp/absl/base/internal/thread_annotations.h deleted file mode 100644 index 8c5c67e0df..0000000000 --- a/contrib/restricted/abseil-cpp/absl/base/internal/thread_annotations.h +++ /dev/null @@ -1,280 +0,0 @@ -// Copyright 2019 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// ----------------------------------------------------------------------------- -// File: thread_annotations.h -// ----------------------------------------------------------------------------- -// -// WARNING: This is a backwards compatible header and it will be removed after -// the migration to prefixed thread annotations is finished; please include -// "absl/base/thread_annotations.h". -// -// This header file contains macro definitions for thread safety annotations -// that allow developers to document the locking policies of multi-threaded -// code. The annotations can also help program analysis tools to identify -// potential thread safety issues. -// -// These annotations are implemented using compiler attributes. Using the macros -// defined here instead of raw attributes allow for portability and future -// compatibility. -// -// When referring to mutexes in the arguments of the attributes, you should -// use variable names or more complex expressions (e.g. my_object->mutex_) -// that evaluate to a concrete mutex object whenever possible. If the mutex -// you want to refer to is not in scope, you may use a member pointer -// (e.g. &MyClass::mutex_) to refer to a mutex in some (unknown) object. - -#ifndef ABSL_BASE_INTERNAL_THREAD_ANNOTATIONS_H_ -#define ABSL_BASE_INTERNAL_THREAD_ANNOTATIONS_H_ - -// ABSL_LEGACY_THREAD_ANNOTATIONS is a *temporary* compatibility macro that can -// be defined on the compile command-line to restore the legacy spellings of the -// thread annotations macros/functions. The macros in this file are available -// under ABSL_ prefixed spellings in absl/base/thread_annotations.h. This macro -// and the legacy spellings will be removed in the future. -#ifdef ABSL_LEGACY_THREAD_ANNOTATIONS - -#if defined(__clang__) -#define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) -#else -#define THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op -#endif - -// GUARDED_BY() -// -// Documents if a shared field or global variable needs to be protected by a -// mutex. GUARDED_BY() allows the user to specify a particular mutex that -// should be held when accessing the annotated variable. -// -// Although this annotation (and PT_GUARDED_BY, below) cannot be applied to -// local variables, a local variable and its associated mutex can often be -// combined into a small class or struct, thereby allowing the annotation. -// -// Example: -// -// class Foo { -// Mutex mu_; -// int p1_ GUARDED_BY(mu_); -// ... -// }; -#define GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x)) - -// PT_GUARDED_BY() -// -// Documents if the memory location pointed to by a pointer should be guarded -// by a mutex when dereferencing the pointer. -// -// Example: -// class Foo { -// Mutex mu_; -// int *p1_ PT_GUARDED_BY(mu_); -// ... -// }; -// -// Note that a pointer variable to a shared memory location could itself be a -// shared variable. -// -// Example: -// -// // `q_`, guarded by `mu1_`, points to a shared memory location that is -// // guarded by `mu2_`: -// int *q_ GUARDED_BY(mu1_) PT_GUARDED_BY(mu2_); -#define PT_GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x)) - -// ACQUIRED_AFTER() / ACQUIRED_BEFORE() -// -// Documents the acquisition order between locks that can be held -// simultaneously by a thread. For any two locks that need to be annotated -// to establish an acquisition order, only one of them needs the annotation. -// (i.e. You don't have to annotate both locks with both ACQUIRED_AFTER -// and ACQUIRED_BEFORE.) -// -// As with GUARDED_BY, this is only applicable to mutexes that are shared -// fields or global variables. -// -// Example: -// -// Mutex m1_; -// Mutex m2_ ACQUIRED_AFTER(m1_); -#define ACQUIRED_AFTER(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__)) - -#define ACQUIRED_BEFORE(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__)) - -// EXCLUSIVE_LOCKS_REQUIRED() / SHARED_LOCKS_REQUIRED() -// -// Documents a function that expects a mutex to be held prior to entry. -// The mutex is expected to be held both on entry to, and exit from, the -// function. -// -// An exclusive lock allows read-write access to the guarded data member(s), and -// only one thread can acquire a lock exclusively at any one time. A shared lock -// allows read-only access, and any number of threads can acquire a shared lock -// concurrently. -// -// Generally, non-const methods should be annotated with -// EXCLUSIVE_LOCKS_REQUIRED, while const methods should be annotated with -// SHARED_LOCKS_REQUIRED. -// -// Example: -// -// Mutex mu1, mu2; -// int a GUARDED_BY(mu1); -// int b GUARDED_BY(mu2); -// -// void foo() EXCLUSIVE_LOCKS_REQUIRED(mu1, mu2) { ... } -// void bar() const SHARED_LOCKS_REQUIRED(mu1, mu2) { ... } -#define EXCLUSIVE_LOCKS_REQUIRED(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(exclusive_locks_required(__VA_ARGS__)) - -#define SHARED_LOCKS_REQUIRED(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(shared_locks_required(__VA_ARGS__)) - -// LOCKS_EXCLUDED() -// -// Documents the locks acquired in the body of the function. These locks -// cannot be held when calling this function (as Abseil's `Mutex` locks are -// non-reentrant). -#define LOCKS_EXCLUDED(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(__VA_ARGS__)) - -// LOCK_RETURNED() -// -// Documents a function that returns a mutex without acquiring it. For example, -// a public getter method that returns a pointer to a private mutex should -// be annotated with LOCK_RETURNED. -#define LOCK_RETURNED(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x)) - -// LOCKABLE -// -// Documents if a class/type is a lockable type (such as the `Mutex` class). -#define LOCKABLE \ - THREAD_ANNOTATION_ATTRIBUTE__(lockable) - -// SCOPED_LOCKABLE -// -// Documents if a class does RAII locking (such as the `MutexLock` class). -// The constructor should use `LOCK_FUNCTION()` to specify the mutex that is -// acquired, and the destructor should use `UNLOCK_FUNCTION()` with no -// arguments; the analysis will assume that the destructor unlocks whatever the -// constructor locked. -#define SCOPED_LOCKABLE \ - THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable) - -// EXCLUSIVE_LOCK_FUNCTION() -// -// Documents functions that acquire a lock in the body of a function, and do -// not release it. -#define EXCLUSIVE_LOCK_FUNCTION(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(exclusive_lock_function(__VA_ARGS__)) - -// SHARED_LOCK_FUNCTION() -// -// Documents functions that acquire a shared (reader) lock in the body of a -// function, and do not release it. -#define SHARED_LOCK_FUNCTION(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(shared_lock_function(__VA_ARGS__)) - -// UNLOCK_FUNCTION() -// -// Documents functions that expect a lock to be held on entry to the function, -// and release it in the body of the function. -#define UNLOCK_FUNCTION(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(unlock_function(__VA_ARGS__)) - -// EXCLUSIVE_TRYLOCK_FUNCTION() / SHARED_TRYLOCK_FUNCTION() -// -// Documents functions that try to acquire a lock, and return success or failure -// (or a non-boolean value that can be interpreted as a boolean). -// The first argument should be `true` for functions that return `true` on -// success, or `false` for functions that return `false` on success. The second -// argument specifies the mutex that is locked on success. If unspecified, this -// mutex is assumed to be `this`. -#define EXCLUSIVE_TRYLOCK_FUNCTION(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(exclusive_trylock_function(__VA_ARGS__)) - -#define SHARED_TRYLOCK_FUNCTION(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(shared_trylock_function(__VA_ARGS__)) - -// ASSERT_EXCLUSIVE_LOCK() / ASSERT_SHARED_LOCK() -// -// Documents functions that dynamically check to see if a lock is held, and fail -// if it is not held. -#define ASSERT_EXCLUSIVE_LOCK(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(assert_exclusive_lock(__VA_ARGS__)) - -#define ASSERT_SHARED_LOCK(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(assert_shared_lock(__VA_ARGS__)) - -// NO_THREAD_SAFETY_ANALYSIS -// -// Turns off thread safety checking within the body of a particular function. -// This annotation is used to mark functions that are known to be correct, but -// the locking behavior is more complicated than the analyzer can handle. -#define NO_THREAD_SAFETY_ANALYSIS \ - THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis) - -//------------------------------------------------------------------------------ -// Tool-Supplied Annotations -//------------------------------------------------------------------------------ - -// TS_UNCHECKED should be placed around lock expressions that are not valid -// C++ syntax, but which are present for documentation purposes. These -// annotations will be ignored by the analysis. -#define TS_UNCHECKED(x) "" - -// TS_FIXME is used to mark lock expressions that are not valid C++ syntax. -// It is used by automated tools to mark and disable invalid expressions. -// The annotation should either be fixed, or changed to TS_UNCHECKED. -#define TS_FIXME(x) "" - -// Like NO_THREAD_SAFETY_ANALYSIS, this turns off checking within the body of -// a particular function. However, this attribute is used to mark functions -// that are incorrect and need to be fixed. It is used by automated tools to -// avoid breaking the build when the analysis is updated. -// Code owners are expected to eventually fix the routine. -#define NO_THREAD_SAFETY_ANALYSIS_FIXME NO_THREAD_SAFETY_ANALYSIS - -// Similar to NO_THREAD_SAFETY_ANALYSIS_FIXME, this macro marks a GUARDED_BY -// annotation that needs to be fixed, because it is producing thread safety -// warning. It disables the GUARDED_BY. -#define GUARDED_BY_FIXME(x) - -// Disables warnings for a single read operation. This can be used to avoid -// warnings when it is known that the read is not actually involved in a race, -// but the compiler cannot confirm that. -#define TS_UNCHECKED_READ(x) thread_safety_analysis::ts_unchecked_read(x) - - -namespace thread_safety_analysis { - -// Takes a reference to a guarded data member, and returns an unguarded -// reference. -template <typename T> -inline const T& ts_unchecked_read(const T& v) NO_THREAD_SAFETY_ANALYSIS { - return v; -} - -template <typename T> -inline T& ts_unchecked_read(T& v) NO_THREAD_SAFETY_ANALYSIS { - return v; -} - -} // namespace thread_safety_analysis - -#endif // defined(ABSL_LEGACY_THREAD_ANNOTATIONS) - -#endif // ABSL_BASE_INTERNAL_THREAD_ANNOTATIONS_H_ diff --git a/contrib/restricted/abseil-cpp/absl/base/internal/thread_identity.cc b/contrib/restricted/abseil-cpp/absl/base/internal/thread_identity.cc index 252443ebcd..0471a25dce 100644 --- a/contrib/restricted/abseil-cpp/absl/base/internal/thread_identity.cc +++ b/contrib/restricted/abseil-cpp/absl/base/internal/thread_identity.cc @@ -16,8 +16,12 @@ #if !defined(_WIN32) || defined(__MINGW32__) #include <pthread.h> +#ifndef __wasi__ +// WASI does not provide this header, either way we disable use +// of signals with it below. #include <signal.h> #endif +#endif #include <atomic> #include <cassert> @@ -80,10 +84,12 @@ void SetCurrentThreadIdentity(ThreadIdentity* identity, absl::call_once(init_thread_identity_key_once, AllocateThreadIdentityKey, reclaimer); -#if defined(__EMSCRIPTEN__) || defined(__MINGW32__) || defined(__hexagon__) - // Emscripten and MinGW pthread implementations does not support signals. - // See https://kripken.github.io/emscripten-site/docs/porting/pthreads.html - // for more information. +#if defined(__wasi__) || defined(__EMSCRIPTEN__) || defined(__MINGW32__) || \ + defined(__hexagon__) + // Emscripten, WASI and MinGW pthread implementations does not support + // signals. See + // https://kripken.github.io/emscripten-site/docs/porting/pthreads.html for + // more information. pthread_setspecific(thread_identity_pthread_key, reinterpret_cast<void*>(identity)); #else diff --git a/contrib/restricted/abseil-cpp/absl/base/internal/unaligned_access.h b/contrib/restricted/abseil-cpp/absl/base/internal/unaligned_access.h index 093dd9b499..4fea457491 100644 --- a/contrib/restricted/abseil-cpp/absl/base/internal/unaligned_access.h +++ b/contrib/restricted/abseil-cpp/absl/base/internal/unaligned_access.h @@ -23,6 +23,7 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" +#include "absl/base/nullability.h" // unaligned APIs @@ -35,29 +36,35 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace base_internal { -inline uint16_t UnalignedLoad16(const void *p) { +inline uint16_t UnalignedLoad16(absl::Nonnull<const void *> p) { uint16_t t; memcpy(&t, p, sizeof t); return t; } -inline uint32_t UnalignedLoad32(const void *p) { +inline uint32_t UnalignedLoad32(absl::Nonnull<const void *> p) { uint32_t t; memcpy(&t, p, sizeof t); return t; } -inline uint64_t UnalignedLoad64(const void *p) { +inline uint64_t UnalignedLoad64(absl::Nonnull<const void *> p) { uint64_t t; memcpy(&t, p, sizeof t); return t; } -inline void UnalignedStore16(void *p, uint16_t v) { memcpy(p, &v, sizeof v); } +inline void UnalignedStore16(absl::Nonnull<void *> p, uint16_t v) { + memcpy(p, &v, sizeof v); +} -inline void UnalignedStore32(void *p, uint32_t v) { memcpy(p, &v, sizeof v); } +inline void UnalignedStore32(absl::Nonnull<void *> p, uint32_t v) { + memcpy(p, &v, sizeof v); +} -inline void UnalignedStore64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); } +inline void UnalignedStore64(absl::Nonnull<void *> p, uint64_t v) { + memcpy(p, &v, sizeof v); +} } // namespace base_internal ABSL_NAMESPACE_END diff --git a/contrib/restricted/abseil-cpp/absl/base/log_severity.cc b/contrib/restricted/abseil-cpp/absl/base/log_severity.cc index 60a8fc1f89..8e7bbbc9e0 100644 --- a/contrib/restricted/abseil-cpp/absl/base/log_severity.cc +++ b/contrib/restricted/abseil-cpp/absl/base/log_severity.cc @@ -17,6 +17,7 @@ #include <ostream> #include "absl/base/attributes.h" +#include "absl/base/config.h" namespace absl { ABSL_NAMESPACE_BEGIN diff --git a/contrib/restricted/abseil-cpp/absl/base/log_severity.h b/contrib/restricted/abseil-cpp/absl/base/log_severity.h index 8bdca38b5f..de9702ab68 100644 --- a/contrib/restricted/abseil-cpp/absl/base/log_severity.h +++ b/contrib/restricted/abseil-cpp/absl/base/log_severity.h @@ -64,6 +64,8 @@ ABSL_NAMESPACE_BEGIN // --my_log_level=info // --my_log_level=0 // +// `DFATAL` and `kLogDebugFatal` are similarly accepted. +// // Unparsing a flag produces the same result as `absl::LogSeverityName()` for // the standard levels and a base-ten integer otherwise. enum class LogSeverity : int { @@ -82,18 +84,28 @@ constexpr std::array<absl::LogSeverity, 4> LogSeverities() { absl::LogSeverity::kError, absl::LogSeverity::kFatal}}; } +// `absl::kLogDebugFatal` equals `absl::LogSeverity::kFatal` in debug builds +// (i.e. when `NDEBUG` is not defined) and `absl::LogSeverity::kError` +// otherwise. Avoid ODR-using this variable as it has internal linkage and thus +// distinct storage in different TUs. +#ifdef NDEBUG +static constexpr absl::LogSeverity kLogDebugFatal = absl::LogSeverity::kError; +#else +static constexpr absl::LogSeverity kLogDebugFatal = absl::LogSeverity::kFatal; +#endif + // LogSeverityName() // // Returns the all-caps string representation (e.g. "INFO") of the specified // severity level if it is one of the standard levels and "UNKNOWN" otherwise. constexpr const char* LogSeverityName(absl::LogSeverity s) { - return s == absl::LogSeverity::kInfo - ? "INFO" - : s == absl::LogSeverity::kWarning - ? "WARNING" - : s == absl::LogSeverity::kError - ? "ERROR" - : s == absl::LogSeverity::kFatal ? "FATAL" : "UNKNOWN"; + switch (s) { + case absl::LogSeverity::kInfo: return "INFO"; + case absl::LogSeverity::kWarning: return "WARNING"; + case absl::LogSeverity::kError: return "ERROR"; + case absl::LogSeverity::kFatal: return "FATAL"; + } + return "UNKNOWN"; } // NormalizeLogSeverity() @@ -101,9 +113,10 @@ constexpr const char* LogSeverityName(absl::LogSeverity s) { // Values less than `kInfo` normalize to `kInfo`; values greater than `kFatal` // normalize to `kError` (**NOT** `kFatal`). constexpr absl::LogSeverity NormalizeLogSeverity(absl::LogSeverity s) { - return s < absl::LogSeverity::kInfo - ? absl::LogSeverity::kInfo - : s > absl::LogSeverity::kFatal ? absl::LogSeverity::kError : s; + absl::LogSeverity n = s; + if (n < absl::LogSeverity::kInfo) n = absl::LogSeverity::kInfo; + if (n > absl::LogSeverity::kFatal) n = absl::LogSeverity::kError; + return n; } constexpr absl::LogSeverity NormalizeLogSeverity(int s) { return absl::NormalizeLogSeverity(static_cast<absl::LogSeverity>(s)); diff --git a/contrib/restricted/abseil-cpp/absl/base/no_destructor.h b/contrib/restricted/abseil-cpp/absl/base/no_destructor.h new file mode 100644 index 0000000000..d4b16a6e88 --- /dev/null +++ b/contrib/restricted/abseil-cpp/absl/base/no_destructor.h @@ -0,0 +1,217 @@ +// Copyright 2023 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: no_destructor.h +// ----------------------------------------------------------------------------- +// +// This header file defines the absl::NoDestructor<T> wrapper for defining a +// static type that does not need to be destructed upon program exit. Instead, +// such an object survives during program exit (and can be safely accessed at +// any time). +// +// Objects of such type, if constructed safely and under the right conditions, +// provide two main benefits over other alternatives: +// +// * Global objects not normally allowed due to concerns of destruction order +// (i.e. no "complex globals") can be safely allowed, provided that such +// objects can be constant initialized. +// * Function scope static objects can be optimized to avoid heap allocation, +// pointer chasing, and allow lazy construction. +// +// See below for complete details. + + +#ifndef ABSL_BASE_NO_DESTRUCTOR_H_ +#define ABSL_BASE_NO_DESTRUCTOR_H_ + +#include <new> +#include <type_traits> +#include <utility> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// absl::NoDestructor<T> +// +// NoDestructor<T> is a wrapper around an object of type T that behaves as an +// object of type T but never calls T's destructor. NoDestructor<T> makes it +// safer and/or more efficient to use such objects in static storage contexts: +// as global or function scope static variables. +// +// An instance of absl::NoDestructor<T> has similar type semantics to an +// instance of T: +// +// * Constructs in the same manner as an object of type T through perfect +// forwarding. +// * Provides pointer/reference semantic access to the object of type T via +// `->`, `*`, and `get()`. +// (Note that `const NoDestructor<T>` works like a pointer to const `T`.) +// +// An object of type NoDestructor<T> should be defined in static storage: +// as either a global static object, or as a function scope static variable. +// +// Additionally, NoDestructor<T> provides the following benefits: +// +// * Never calls T's destructor for the object +// * If the object is a function-local static variable, the type can be +// lazily constructed. +// +// An object of type NoDestructor<T> is "trivially destructible" in the notion +// that its destructor is never run. Provided that an object of this type can be +// safely initialized and does not need to be cleaned up on program shutdown, +// NoDestructor<T> allows you to define global static variables, since Google's +// C++ style guide ban on such objects doesn't apply to objects that are +// trivially destructible. +// +// Usage as Global Static Variables +// +// NoDestructor<T> allows declaration of a global object with a non-trivial +// constructor in static storage without needing to add a destructor. +// However, such objects still need to worry about initialization order, so +// such objects should be const initialized: +// +// // Global or namespace scope. +// ABSL_CONST_INIT absl::NoDestructor<MyRegistry> reg{"foo", "bar", 8008}; +// +// Note that if your object already has a trivial destructor, you don't need to +// use NoDestructor<T>. +// +// Usage as Function Scope Static Variables +// +// Function static objects will be lazily initialized within static storage: +// +// // Function scope. +// const std::string& MyString() { +// static const absl::NoDestructor<std::string> x("foo"); +// return *x; +// } +// +// For function static variables, NoDestructor avoids heap allocation and can be +// inlined in static storage, resulting in exactly-once, thread-safe +// construction of an object, and very fast access thereafter (the cost is a few +// extra cycles). +// +// Using NoDestructor<T> in this manner is generally better than other patterns +// which require pointer chasing: +// +// // Prefer using absl::NoDestructor<T> instead for the static variable. +// const std::string& MyString() { +// static const std::string* x = new std::string("foo"); +// return *x; +// } +// +template <typename T> +class NoDestructor { + public: + // Forwards arguments to the T's constructor: calls T(args...). + template <typename... Ts, + // Disable this overload when it might collide with copy/move. + typename std::enable_if<!std::is_same<void(std::decay_t<Ts>&...), + void(NoDestructor&)>::value, + int>::type = 0> + explicit constexpr NoDestructor(Ts&&... args) + : impl_(std::forward<Ts>(args)...) {} + + // Forwards copy and move construction for T. Enables usage like this: + // static NoDestructor<std::array<string, 3>> x{{{"1", "2", "3"}}}; + // static NoDestructor<std::vector<int>> x{{1, 2, 3}}; + explicit constexpr NoDestructor(const T& x) : impl_(x) {} + explicit constexpr NoDestructor(T&& x) + : impl_(std::move(x)) {} + + // No copying. + NoDestructor(const NoDestructor&) = delete; + NoDestructor& operator=(const NoDestructor&) = delete; + + // Pretend to be a smart pointer to T with deep constness. + // Never returns a null pointer. + T& operator*() { return *get(); } + T* operator->() { return get(); } + T* get() { return impl_.get(); } + const T& operator*() const { return *get(); } + const T* operator->() const { return get(); } + const T* get() const { return impl_.get(); } + + private: + class DirectImpl { + public: + template <typename... Args> + explicit constexpr DirectImpl(Args&&... args) + : value_(std::forward<Args>(args)...) {} + const T* get() const { return &value_; } + T* get() { return &value_; } + + private: + T value_; + }; + + class PlacementImpl { + public: + template <typename... Args> + explicit PlacementImpl(Args&&... args) { + new (&space_) T(std::forward<Args>(args)...); + } + const T* get() const { + return Launder(reinterpret_cast<const T*>(&space_)); + } + T* get() { return Launder(reinterpret_cast<T*>(&space_)); } + + private: + template <typename P> + static P* Launder(P* p) { +#if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606L + return std::launder(p); +#elif ABSL_HAVE_BUILTIN(__builtin_launder) + return __builtin_launder(p); +#else + // When `std::launder` or equivalent are not available, we rely on + // undefined behavior, which works as intended on Abseil's officially + // supported platforms as of Q3 2023. +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wstrict-aliasing" +#endif + return p; +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif +#endif + } + + alignas(T) unsigned char space_[sizeof(T)]; + }; + + // If the object is trivially destructible we use a member directly to avoid + // potential once-init runtime initialization. It somewhat defeats the + // purpose of NoDestructor in this case, but this makes the class more + // friendly to generic code. + std::conditional_t<std::is_trivially_destructible<T>::value, DirectImpl, + PlacementImpl> + impl_; +}; + +#ifdef ABSL_HAVE_CLASS_TEMPLATE_ARGUMENT_DEDUCTION +// Provide 'Class Template Argument Deduction': the type of NoDestructor's T +// will be the same type as the argument passed to NoDestructor's constructor. +template <typename T> +NoDestructor(T) -> NoDestructor<T>; +#endif // ABSL_HAVE_CLASS_TEMPLATE_ARGUMENT_DEDUCTION + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_NO_DESTRUCTOR_H_ diff --git a/contrib/restricted/abseil-cpp/absl/base/nullability.h b/contrib/restricted/abseil-cpp/absl/base/nullability.h new file mode 100644 index 0000000000..6f49b6f535 --- /dev/null +++ b/contrib/restricted/abseil-cpp/absl/base/nullability.h @@ -0,0 +1,224 @@ +// Copyright 2023 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: nullability.h +// ----------------------------------------------------------------------------- +// +// This header file defines a set of "templated annotations" for designating the +// expected nullability of pointers. These annotations allow you to designate +// pointers in one of three classification states: +// +// * "Non-null" (for pointers annotated `Nonnull<T>`), indicating that it is +// invalid for the given pointer to ever be null. +// * "Nullable" (for pointers annotated `Nullable<T>`), indicating that it is +// valid for the given pointer to be null. +// * "Unknown" (for pointers annotated `NullabilityUnknown<T>`), indicating +// that the given pointer has not been yet classified as either nullable or +// non-null. This is the default state of unannotated pointers. +// +// NOTE: unannotated pointers implicitly bear the annotation +// `NullabilityUnknown<T>`; you should rarely, if ever, see this annotation used +// in the codebase explicitly. +// +// ----------------------------------------------------------------------------- +// Nullability and Contracts +// ----------------------------------------------------------------------------- +// +// These nullability annotations allow you to more clearly specify contracts on +// software components by narrowing the *preconditions*, *postconditions*, and +// *invariants* of pointer state(s) in any given interface. It then depends on +// context who is responsible for fulfilling the annotation's requirements. +// +// For example, a function may receive a pointer argument. Designating that +// pointer argument as "non-null" tightens the precondition of the contract of +// that function. It is then the responsibility of anyone calling such a +// function to ensure that the passed pointer is not null. +// +// Similarly, a function may have a pointer as a return value. Designating that +// return value as "non-null" tightens the postcondition of the contract of that +// function. In this case, however, it is the responsibility of the function +// itself to ensure that the returned pointer is not null. +// +// Clearly defining these contracts allows providers (and consumers) of such +// pointers to have more confidence in their null state. If a function declares +// a return value as "non-null", for example, the caller should not need to +// check whether the returned value is `nullptr`; it can simply assume the +// pointer is valid. +// +// Of course most interfaces already have expectations on the nullability state +// of pointers, and these expectations are, in effect, a contract; often, +// however, those contracts are either poorly or partially specified, assumed, +// or misunderstood. These nullability annotations are designed to allow you to +// formalize those contracts within the codebase. +// +// ----------------------------------------------------------------------------- +// Using Nullability Annotations +// ----------------------------------------------------------------------------- +// +// It is important to note that these annotations are not distinct strong +// *types*. They are alias templates defined to be equal to the underlying +// pointer type. A pointer annotated `Nonnull<T*>`, for example, is simply a +// pointer of type `T*`. Each annotation acts as a form of documentation about +// the contract for the given pointer. Each annotation requires providers or +// consumers of these pointers across API boundaries to take appropriate steps +// when setting or using these pointers: +// +// * "Non-null" pointers should never be null. It is the responsibility of the +// provider of this pointer to ensure that the pointer may never be set to +// null. Consumers of such pointers can treat such pointers as non-null. +// * "Nullable" pointers may or may not be null. Consumers of such pointers +// should precede any usage of that pointer (e.g. a dereference operation) +// with a a `nullptr` check. +// * "Unknown" pointers may be either "non-null" or "nullable" but have not been +// definitively determined to be in either classification state. Providers of +// such pointers across API boundaries should determine -- over time -- to +// annotate the pointer in either of the above two states. Consumers of such +// pointers across an API boundary should continue to treat such pointers as +// they currently do. +// +// Example: +// +// // PaySalary() requires the passed pointer to an `Employee` to be non-null. +// void PaySalary(absl::Nonnull<Employee *> e) { +// pay(e->salary); // OK to dereference +// } +// +// // CompleteTransaction() guarantees the returned pointer to an `Account` to +// // be non-null. +// absl::Nonnull<Account *> balance CompleteTransaction(double fee) { +// ... +// } +// +// // Note that specifying a nullability annotation does not prevent someone +// // from violating the contract: +// +// Nullable<Employee *> find(Map& employees, std::string_view name); +// +// void g(Map& employees) { +// Employee *e = find(employees, "Pat"); +// // `e` can now be null. +// PaySalary(e); // Violates contract, but compiles! +// } +// +// Nullability annotations, in other words, are useful for defining and +// narrowing contracts; *enforcement* of those contracts depends on use and any +// additional (static or dynamic analysis) tooling. +// +// NOTE: The "unknown" annotation state indicates that a pointer's contract has +// not yet been positively identified. The unknown state therefore acts as a +// form of documentation of your technical debt, and a codebase that adopts +// nullability annotations should aspire to annotate every pointer as either +// "non-null" or "nullable". +// +// ----------------------------------------------------------------------------- +// Applicability of Nullability Annotations +// ----------------------------------------------------------------------------- +// +// By default, nullability annotations are applicable to raw and smart +// pointers. User-defined types can indicate compatibility with nullability +// annotations by providing an `absl_nullability_compatible` nested type. The +// actual definition of this inner type is not relevant as it is used merely as +// a marker. It is common to use a using declaration of +// `absl_nullability_compatible` set to void. +// +// // Example: +// struct MyPtr { +// using absl_nullability_compatible = void; +// ... +// }; +// +// DISCLAIMER: +// =========================================================================== +// These nullability annotations are primarily a human readable signal about the +// intended contract of the pointer. They are not *types* and do not currently +// provide any correctness guarantees. For example, a pointer annotated as +// `Nonnull<T*>` is *not guaranteed* to be non-null, and the compiler won't +// alert or prevent assignment of a `Nullable<T*>` to a `Nonnull<T*>`. +// =========================================================================== +#ifndef ABSL_BASE_NULLABILITY_H_ +#define ABSL_BASE_NULLABILITY_H_ + +#include "absl/base/internal/nullability_impl.h" + +namespace absl { + +// absl::Nonnull +// +// The indicated pointer is never null. It is the responsibility of the provider +// of this pointer across an API boundary to ensure that the pointer is never be +// set to null. Consumers of this pointer across an API boundary may safely +// dereference the pointer. +// +// Example: +// +// // `employee` is designated as not null. +// void PaySalary(absl::Nonnull<Employee *> employee) { +// pay(*employee); // OK to dereference +// } +template <typename T> +using Nonnull = nullability_internal::NonnullImpl<T>; + +// absl::Nullable +// +// The indicated pointer may, by design, be either null or non-null. Consumers +// of this pointer across an API boundary should perform a `nullptr` check +// before performing any operation using the pointer. +// +// Example: +// +// // `employee` may be null. +// void PaySalary(absl::Nullable<Employee *> employee) { +// if (employee != nullptr) { +// Pay(*employee); // OK to dereference +// } +// } +template <typename T> +using Nullable = nullability_internal::NullableImpl<T>; + +// absl::NullabilityUnknown (default) +// +// The indicated pointer has not yet been determined to be definitively +// "non-null" or "nullable." Providers of such pointers across API boundaries +// should, over time, annotate such pointers as either "non-null" or "nullable." +// Consumers of these pointers across an API boundary should treat such pointers +// with the same caution they treat currently unannotated pointers. Most +// existing code will have "unknown" pointers, which should eventually be +// migrated into one of the above two nullability states: `Nonnull<T>` or +// `Nullable<T>`. +// +// NOTE: Because this annotation is the global default state, pointers without +// any annotation are assumed to have "unknown" semantics. This assumption is +// designed to minimize churn and reduce clutter within the codebase. +// +// Example: +// +// // `employee`s nullability state is unknown. +// void PaySalary(absl::NullabilityUnknown<Employee *> employee) { +// Pay(*employee); // Potentially dangerous. API provider should investigate. +// } +// +// Note that a pointer without an annotation, by default, is assumed to have the +// annotation `NullabilityUnknown`. +// +// // `employee`s nullability state is unknown. +// void PaySalary(Employee* employee) { +// Pay(*employee); // Potentially dangerous. API provider should investigate. +// } +template <typename T> +using NullabilityUnknown = nullability_internal::NullabilityUnknownImpl<T>; + +} // namespace absl + +#endif // ABSL_BASE_NULLABILITY_H_ diff --git a/contrib/restricted/abseil-cpp/absl/base/optimization.h b/contrib/restricted/abseil-cpp/absl/base/optimization.h index ad0121ad3b..f985995899 100644 --- a/contrib/restricted/abseil-cpp/absl/base/optimization.h +++ b/contrib/restricted/abseil-cpp/absl/base/optimization.h @@ -25,6 +25,7 @@ #include <assert.h> #include "absl/base/config.h" +#include "absl/base/options.h" // ABSL_BLOCK_TAIL_CALL_OPTIMIZATION // diff --git a/contrib/restricted/abseil-cpp/absl/base/options.h b/contrib/restricted/abseil-cpp/absl/base/options.h index f308e1bf8b..d7bf8cffe6 100644 --- a/contrib/restricted/abseil-cpp/absl/base/options.h +++ b/contrib/restricted/abseil-cpp/absl/base/options.h @@ -176,6 +176,32 @@ #define ABSL_OPTION_USE_STD_VARIANT 2 +// ABSL_OPTION_USE_STD_ORDERING +// +// This option controls whether absl::{partial,weak,strong}_ordering are +// implemented as aliases to the std:: ordering types, or as an independent +// implementation. +// +// A value of 0 means to use Abseil's implementation. This requires only C++11 +// support, and is expected to work on every toolchain we support. +// +// A value of 1 means to use aliases. This requires that all code using Abseil +// is built in C++20 mode or later. +// +// A value of 2 means to detect the C++ version being used to compile Abseil, +// and use an alias only if working std:: ordering types are available. This +// option is useful when you are building your program from source. It should +// not be used otherwise -- for example, if you are distributing Abseil in a +// binary package manager -- since in mode 2, they will name different types, +// with different mangled names and binary layout, depending on the compiler +// flags passed by the end user. For more info, see +// https://abseil.io/about/design/dropin-types. +// +// User code should not inspect this macro. To check in the preprocessor if +// the ordering types are aliases of std:: ordering types, use the feature macro +// ABSL_USES_STD_ORDERING. + +#define ABSL_OPTION_USE_STD_ORDERING 2 // ABSL_OPTION_USE_INLINE_NAMESPACE // ABSL_OPTION_INLINE_NAMESPACE_NAME @@ -200,7 +226,7 @@ // allowed. #define ABSL_OPTION_USE_INLINE_NAMESPACE 1 -#define ABSL_OPTION_INLINE_NAMESPACE_NAME lts_20230802 +#define ABSL_OPTION_INLINE_NAMESPACE_NAME lts_20240116 // ABSL_OPTION_HARDENED // diff --git a/contrib/restricted/abseil-cpp/absl/base/prefetch.h b/contrib/restricted/abseil-cpp/absl/base/prefetch.h index de7a180d60..eb40a4453c 100644 --- a/contrib/restricted/abseil-cpp/absl/base/prefetch.h +++ b/contrib/restricted/abseil-cpp/absl/base/prefetch.h @@ -24,17 +24,19 @@ #ifndef ABSL_BASE_PREFETCH_H_ #define ABSL_BASE_PREFETCH_H_ +#include "absl/base/attributes.h" #include "absl/base/config.h" #if defined(ABSL_INTERNAL_HAVE_SSE) #include <xmmintrin.h> #endif -#if defined(_MSC_VER) && _MSC_VER >= 1900 && \ - (defined(_M_X64) || defined(_M_IX86)) +#if defined(_MSC_VER) #include <intrin.h> +#if defined(ABSL_INTERNAL_HAVE_SSE) #pragma intrinsic(_mm_prefetch) #endif +#endif namespace absl { ABSL_NAMESPACE_BEGIN @@ -140,21 +142,24 @@ void PrefetchToLocalCacheForWrite(const void* addr); // See __builtin_prefetch: // https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html. // -inline void PrefetchToLocalCache(const void* addr) { +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCache( + const void* addr) { __builtin_prefetch(addr, 0, 3); } -inline void PrefetchToLocalCacheNta(const void* addr) { +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCacheNta( + const void* addr) { __builtin_prefetch(addr, 0, 0); } -inline void PrefetchToLocalCacheForWrite(const void* addr) { +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCacheForWrite( + const void* addr) { // [x86] gcc/clang don't generate PREFETCHW for __builtin_prefetch(.., 1) // unless -march=broadwell or newer; this is not generally the default, so we // manually emit prefetchw. PREFETCHW is recognized as a no-op on older Intel // processors and has been present on AMD processors since the K6-2. -#if defined(__x86_64__) - asm("prefetchw (%0)" : : "r"(addr)); +#if defined(__x86_64__) && !defined(__PRFCHW__) + asm("prefetchw %0" : : "m"(*reinterpret_cast<const char*>(addr))); #else __builtin_prefetch(addr, 1, 3); #endif @@ -164,15 +169,18 @@ inline void PrefetchToLocalCacheForWrite(const void* addr) { #define ABSL_HAVE_PREFETCH 1 -inline void PrefetchToLocalCache(const void* addr) { +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCache( + const void* addr) { _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_T0); } -inline void PrefetchToLocalCacheNta(const void* addr) { +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCacheNta( + const void* addr) { _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_NTA); } -inline void PrefetchToLocalCacheForWrite(const void* addr) { +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCacheForWrite( + const void* addr) { #if defined(_MM_HINT_ET0) _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_ET0); #elif !defined(_MSC_VER) && defined(__x86_64__) @@ -180,15 +188,18 @@ inline void PrefetchToLocalCacheForWrite(const void* addr) { // up, PREFETCHW is recognized as a no-op on older Intel processors // and has been present on AMD processors since the K6-2. We have this // disabled for MSVC compilers as this miscompiles on older MSVC compilers. - asm("prefetchw (%0)" : : "r"(addr)); + asm("prefetchw %0" : : "m"(*reinterpret_cast<const char*>(addr))); #endif } #else -inline void PrefetchToLocalCache(const void* addr) {} -inline void PrefetchToLocalCacheNta(const void* addr) {} -inline void PrefetchToLocalCacheForWrite(const void* addr) {} +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCache( + const void* addr) {} +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCacheNta( + const void* addr) {} +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCacheForWrite( + const void* addr) {} #endif diff --git a/contrib/restricted/abseil-cpp/absl/base/thread_annotations.h b/contrib/restricted/abseil-cpp/absl/base/thread_annotations.h index bc8a620347..4a3f3e33e9 100644 --- a/contrib/restricted/abseil-cpp/absl/base/thread_annotations.h +++ b/contrib/restricted/abseil-cpp/absl/base/thread_annotations.h @@ -36,8 +36,6 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" -// TODO(mbonadei): Remove after the backward compatibility period. -#include "absl/base/internal/thread_annotations.h" // IWYU pragma: export // ABSL_GUARDED_BY() // diff --git a/contrib/restricted/abseil-cpp/absl/container/flat_hash_map.h b/contrib/restricted/abseil-cpp/absl/container/flat_hash_map.h index 8f4d993911..acd013b026 100644 --- a/contrib/restricted/abseil-cpp/absl/container/flat_hash_map.h +++ b/contrib/restricted/abseil-cpp/absl/container/flat_hash_map.h @@ -64,7 +64,7 @@ struct FlatHashMapPolicy; // `insert()`, provided that the map is provided a compatible heterogeneous // hashing function and equality operator. // * Invalidates any references and pointers to elements within the table after -// `rehash()`. +// `rehash()` and when the table is moved. // * Contains a `capacity()` member function indicating the number of element // slots (open, deleted, and empty) within the hash map. // * Returns `void` from the `erase(iterator)` overload. @@ -579,9 +579,9 @@ struct FlatHashMapPolicy { } template <class Allocator> - static void transfer(Allocator* alloc, slot_type* new_slot, + static auto transfer(Allocator* alloc, slot_type* new_slot, slot_type* old_slot) { - slot_policy::transfer(alloc, new_slot, old_slot); + return slot_policy::transfer(alloc, new_slot, old_slot); } template <class F, class... Args> diff --git a/contrib/restricted/abseil-cpp/absl/container/flat_hash_set.h b/contrib/restricted/abseil-cpp/absl/container/flat_hash_set.h index c789c7ef54..a94a82a0bf 100644 --- a/contrib/restricted/abseil-cpp/absl/container/flat_hash_set.h +++ b/contrib/restricted/abseil-cpp/absl/container/flat_hash_set.h @@ -60,7 +60,7 @@ struct FlatHashSetPolicy; // that the set is provided a compatible heterogeneous hashing function and // equality operator. // * Invalidates any references and pointers to elements within the table after -// `rehash()`. +// `rehash()` and when the table is moved. // * Contains a `capacity()` member function indicating the number of element // slots (open, deleted, and empty) within the hash set. // * Returns `void` from the `erase(iterator)` overload. diff --git a/contrib/restricted/abseil-cpp/absl/container/internal/common_policy_traits.h b/contrib/restricted/abseil-cpp/absl/container/internal/common_policy_traits.h index 3558a5439b..57eac678fa 100644 --- a/contrib/restricted/abseil-cpp/absl/container/internal/common_policy_traits.h +++ b/contrib/restricted/abseil-cpp/absl/container/internal/common_policy_traits.h @@ -93,11 +93,13 @@ struct common_policy_traits { struct Rank0 : Rank1 {}; // Use auto -> decltype as an enabler. + // P::transfer returns std::true_type if transfer uses memcpy (e.g. in + // node_slot_policy). template <class Alloc, class P = Policy> static auto transfer_impl(Alloc* alloc, slot_type* new_slot, slot_type* old_slot, Rank0) - -> decltype((void)P::transfer(alloc, new_slot, old_slot)) { - P::transfer(alloc, new_slot, old_slot); + -> decltype(P::transfer(alloc, new_slot, old_slot)) { + return P::transfer(alloc, new_slot, old_slot); } #if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606 // This overload returns true_type for the trait below. diff --git a/contrib/restricted/abseil-cpp/absl/container/internal/container_memory.h b/contrib/restricted/abseil-cpp/absl/container/internal/container_memory.h index f59ca4ee22..3262d4eb8d 100644 --- a/contrib/restricted/abseil-cpp/absl/container/internal/container_memory.h +++ b/contrib/restricted/abseil-cpp/absl/container/internal/container_memory.h @@ -122,10 +122,10 @@ auto TupleRefImpl(T&& t, absl::index_sequence<Is...>) // Returns a tuple of references to the elements of the input tuple. T must be a // tuple. template <class T> -auto TupleRef(T&& t) -> decltype( - TupleRefImpl(std::forward<T>(t), - absl::make_index_sequence< - std::tuple_size<typename std::decay<T>::type>::value>())) { +auto TupleRef(T&& t) -> decltype(TupleRefImpl( + std::forward<T>(t), + absl::make_index_sequence< + std::tuple_size<typename std::decay<T>::type>::value>())) { return TupleRefImpl( std::forward<T>(t), absl::make_index_sequence< @@ -156,8 +156,8 @@ void ConstructFromTuple(Alloc* alloc, T* ptr, Tuple&& t) { // Constructs T using the args specified in the tuple and calls F with the // constructed value. template <class T, class Tuple, class F> -decltype(std::declval<F>()(std::declval<T>())) WithConstructed( - Tuple&& t, F&& f) { +decltype(std::declval<F>()(std::declval<T>())) WithConstructed(Tuple&& t, + F&& f) { return memory_internal::WithConstructedImpl<T>( std::forward<Tuple>(t), absl::make_index_sequence< @@ -423,16 +423,19 @@ struct map_slot_policy { } template <class Allocator> - static void transfer(Allocator* alloc, slot_type* new_slot, + static auto transfer(Allocator* alloc, slot_type* new_slot, slot_type* old_slot) { + auto is_relocatable = + typename absl::is_trivially_relocatable<value_type>::type(); + emplace(new_slot); #if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606 - if (absl::is_trivially_relocatable<value_type>()) { + if (is_relocatable) { // TODO(b/247130232,b/251814870): remove casts after fixing warnings. std::memcpy(static_cast<void*>(std::launder(&new_slot->value)), static_cast<const void*>(&old_slot->value), sizeof(value_type)); - return; + return is_relocatable; } #endif @@ -444,6 +447,7 @@ struct map_slot_policy { std::move(old_slot->value)); } destroy(alloc, old_slot); + return is_relocatable; } }; diff --git a/contrib/restricted/abseil-cpp/absl/container/internal/hashtablez_sampler.h b/contrib/restricted/abseil-cpp/absl/container/internal/hashtablez_sampler.h index d8fd8f34f9..e41ee2d747 100644 --- a/contrib/restricted/abseil-cpp/absl/container/internal/hashtablez_sampler.h +++ b/contrib/restricted/abseil-cpp/absl/container/internal/hashtablez_sampler.h @@ -137,18 +137,7 @@ class HashtablezInfoHandle { UnsampleSlow(info_); } - HashtablezInfoHandle(const HashtablezInfoHandle&) = delete; - HashtablezInfoHandle& operator=(const HashtablezInfoHandle&) = delete; - - HashtablezInfoHandle(HashtablezInfoHandle&& o) noexcept - : info_(absl::exchange(o.info_, nullptr)) {} - HashtablezInfoHandle& operator=(HashtablezInfoHandle&& o) noexcept { - if (ABSL_PREDICT_FALSE(info_ != nullptr)) { - UnsampleSlow(info_); - } - info_ = absl::exchange(o.info_, nullptr); - return *this; - } + inline bool IsSampled() const { return ABSL_PREDICT_FALSE(info_ != nullptr); } inline void RecordStorageChanged(size_t size, size_t capacity) { if (ABSL_PREDICT_TRUE(info_ == nullptr)) return; @@ -198,6 +187,7 @@ class HashtablezInfoHandle { explicit HashtablezInfoHandle(std::nullptr_t) {} inline void Unregister() {} + inline bool IsSampled() const { return false; } inline void RecordStorageChanged(size_t /*size*/, size_t /*capacity*/) {} inline void RecordRehash(size_t /*total_probe_length*/) {} inline void RecordReservation(size_t /*target_capacity*/) {} diff --git a/contrib/restricted/abseil-cpp/absl/container/internal/inlined_vector.h b/contrib/restricted/abseil-cpp/absl/container/internal/inlined_vector.h index b2a602db7e..0eb9c34ddd 100644 --- a/contrib/restricted/abseil-cpp/absl/container/internal/inlined_vector.h +++ b/contrib/restricted/abseil-cpp/absl/container/internal/inlined_vector.h @@ -26,6 +26,7 @@ #include <utility> #include "absl/base/attributes.h" +#include "absl/base/config.h" #include "absl/base/macros.h" #include "absl/container/internal/compressed_tuple.h" #include "absl/memory/memory.h" @@ -384,7 +385,17 @@ class Storage { bool GetIsAllocated() const { return GetSizeAndIsAllocated() & 1; } - Pointer<A> GetAllocatedData() { return data_.allocated.allocated_data; } + Pointer<A> GetAllocatedData() { + // GCC 12 has a false-positive -Wmaybe-uninitialized warning here. +#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(12, 0) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + return data_.allocated.allocated_data; +#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(12, 0) +#pragma GCC diagnostic pop +#endif + } ConstPointer<A> GetAllocatedData() const { return data_.allocated.allocated_data; diff --git a/contrib/restricted/abseil-cpp/absl/container/internal/layout.h b/contrib/restricted/abseil-cpp/absl/container/internal/layout.h deleted file mode 100644 index a59a243059..0000000000 --- a/contrib/restricted/abseil-cpp/absl/container/internal/layout.h +++ /dev/null @@ -1,743 +0,0 @@ -// Copyright 2018 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// MOTIVATION AND TUTORIAL -// -// If you want to put in a single heap allocation N doubles followed by M ints, -// it's easy if N and M are known at compile time. -// -// struct S { -// double a[N]; -// int b[M]; -// }; -// -// S* p = new S; -// -// But what if N and M are known only in run time? Class template Layout to the -// rescue! It's a portable generalization of the technique known as struct hack. -// -// // This object will tell us everything we need to know about the memory -// // layout of double[N] followed by int[M]. It's structurally identical to -// // size_t[2] that stores N and M. It's very cheap to create. -// const Layout<double, int> layout(N, M); -// -// // Allocate enough memory for both arrays. `AllocSize()` tells us how much -// // memory is needed. We are free to use any allocation function we want as -// // long as it returns aligned memory. -// std::unique_ptr<unsigned char[]> p(new unsigned char[layout.AllocSize()]); -// -// // Obtain the pointer to the array of doubles. -// // Equivalent to `reinterpret_cast<double*>(p.get())`. -// // -// // We could have written layout.Pointer<0>(p) instead. If all the types are -// // unique you can use either form, but if some types are repeated you must -// // use the index form. -// double* a = layout.Pointer<double>(p.get()); -// -// // Obtain the pointer to the array of ints. -// // Equivalent to `reinterpret_cast<int*>(p.get() + N * 8)`. -// int* b = layout.Pointer<int>(p); -// -// If we are unable to specify sizes of all fields, we can pass as many sizes as -// we can to `Partial()`. In return, it'll allow us to access the fields whose -// locations and sizes can be computed from the provided information. -// `Partial()` comes in handy when the array sizes are embedded into the -// allocation. -// -// // size_t[1] containing N, size_t[1] containing M, double[N], int[M]. -// using L = Layout<size_t, size_t, double, int>; -// -// unsigned char* Allocate(size_t n, size_t m) { -// const L layout(1, 1, n, m); -// unsigned char* p = new unsigned char[layout.AllocSize()]; -// *layout.Pointer<0>(p) = n; -// *layout.Pointer<1>(p) = m; -// return p; -// } -// -// void Use(unsigned char* p) { -// // First, extract N and M. -// // Specify that the first array has only one element. Using `prefix` we -// // can access the first two arrays but not more. -// constexpr auto prefix = L::Partial(1); -// size_t n = *prefix.Pointer<0>(p); -// size_t m = *prefix.Pointer<1>(p); -// -// // Now we can get pointers to the payload. -// const L layout(1, 1, n, m); -// double* a = layout.Pointer<double>(p); -// int* b = layout.Pointer<int>(p); -// } -// -// The layout we used above combines fixed-size with dynamically-sized fields. -// This is quite common. Layout is optimized for this use case and generates -// optimal code. All computations that can be performed at compile time are -// indeed performed at compile time. -// -// Efficiency tip: The order of fields matters. In `Layout<T1, ..., TN>` try to -// ensure that `alignof(T1) >= ... >= alignof(TN)`. This way you'll have no -// padding in between arrays. -// -// You can manually override the alignment of an array by wrapping the type in -// `Aligned<T, N>`. `Layout<..., Aligned<T, N>, ...>` has exactly the same API -// and behavior as `Layout<..., T, ...>` except that the first element of the -// array of `T` is aligned to `N` (the rest of the elements follow without -// padding). `N` cannot be less than `alignof(T)`. -// -// `AllocSize()` and `Pointer()` are the most basic methods for dealing with -// memory layouts. Check out the reference or code below to discover more. -// -// EXAMPLE -// -// // Immutable move-only string with sizeof equal to sizeof(void*). The -// // string size and the characters are kept in the same heap allocation. -// class CompactString { -// public: -// CompactString(const char* s = "") { -// const size_t size = strlen(s); -// // size_t[1] followed by char[size + 1]. -// const L layout(1, size + 1); -// p_.reset(new unsigned char[layout.AllocSize()]); -// // If running under ASAN, mark the padding bytes, if any, to catch -// // memory errors. -// layout.PoisonPadding(p_.get()); -// // Store the size in the allocation. -// *layout.Pointer<size_t>(p_.get()) = size; -// // Store the characters in the allocation. -// memcpy(layout.Pointer<char>(p_.get()), s, size + 1); -// } -// -// size_t size() const { -// // Equivalent to reinterpret_cast<size_t&>(*p). -// return *L::Partial().Pointer<size_t>(p_.get()); -// } -// -// const char* c_str() const { -// // Equivalent to reinterpret_cast<char*>(p.get() + sizeof(size_t)). -// // The argument in Partial(1) specifies that we have size_t[1] in front -// // of the characters. -// return L::Partial(1).Pointer<char>(p_.get()); -// } -// -// private: -// // Our heap allocation contains a size_t followed by an array of chars. -// using L = Layout<size_t, char>; -// std::unique_ptr<unsigned char[]> p_; -// }; -// -// int main() { -// CompactString s = "hello"; -// assert(s.size() == 5); -// assert(strcmp(s.c_str(), "hello") == 0); -// } -// -// DOCUMENTATION -// -// The interface exported by this file consists of: -// - class `Layout<>` and its public members. -// - The public members of class `internal_layout::LayoutImpl<>`. That class -// isn't intended to be used directly, and its name and template parameter -// list are internal implementation details, but the class itself provides -// most of the functionality in this file. See comments on its members for -// detailed documentation. -// -// `Layout<T1,... Tn>::Partial(count1,..., countm)` (where `m` <= `n`) returns a -// `LayoutImpl<>` object. `Layout<T1,..., Tn> layout(count1,..., countn)` -// creates a `Layout` object, which exposes the same functionality by inheriting -// from `LayoutImpl<>`. - -#ifndef ABSL_CONTAINER_INTERNAL_LAYOUT_H_ -#define ABSL_CONTAINER_INTERNAL_LAYOUT_H_ - -#include <assert.h> -#include <stddef.h> -#include <stdint.h> - -#include <ostream> -#include <string> -#include <tuple> -#include <type_traits> -#include <typeinfo> -#include <utility> - -#include "absl/base/config.h" -#include "absl/meta/type_traits.h" -#include "absl/strings/str_cat.h" -#include "absl/types/span.h" -#include "absl/utility/utility.h" - -#ifdef ABSL_HAVE_ADDRESS_SANITIZER -#include <sanitizer/asan_interface.h> -#endif - -#if defined(__GXX_RTTI) -#define ABSL_INTERNAL_HAS_CXA_DEMANGLE -#endif - -#ifdef ABSL_INTERNAL_HAS_CXA_DEMANGLE -#include <cxxabi.h> -#endif - -namespace absl { -ABSL_NAMESPACE_BEGIN -namespace container_internal { - -// A type wrapper that instructs `Layout` to use the specific alignment for the -// array. `Layout<..., Aligned<T, N>, ...>` has exactly the same API -// and behavior as `Layout<..., T, ...>` except that the first element of the -// array of `T` is aligned to `N` (the rest of the elements follow without -// padding). -// -// Requires: `N >= alignof(T)` and `N` is a power of 2. -template <class T, size_t N> -struct Aligned; - -namespace internal_layout { - -template <class T> -struct NotAligned {}; - -template <class T, size_t N> -struct NotAligned<const Aligned<T, N>> { - static_assert(sizeof(T) == 0, "Aligned<T, N> cannot be const-qualified"); -}; - -template <size_t> -using IntToSize = size_t; - -template <class> -using TypeToSize = size_t; - -template <class T> -struct Type : NotAligned<T> { - using type = T; -}; - -template <class T, size_t N> -struct Type<Aligned<T, N>> { - using type = T; -}; - -template <class T> -struct SizeOf : NotAligned<T>, std::integral_constant<size_t, sizeof(T)> {}; - -template <class T, size_t N> -struct SizeOf<Aligned<T, N>> : std::integral_constant<size_t, sizeof(T)> {}; - -// Note: workaround for https://gcc.gnu.org/PR88115 -template <class T> -struct AlignOf : NotAligned<T> { - static constexpr size_t value = alignof(T); -}; - -template <class T, size_t N> -struct AlignOf<Aligned<T, N>> { - static_assert(N % alignof(T) == 0, - "Custom alignment can't be lower than the type's alignment"); - static constexpr size_t value = N; -}; - -// Does `Ts...` contain `T`? -template <class T, class... Ts> -using Contains = absl::disjunction<std::is_same<T, Ts>...>; - -template <class From, class To> -using CopyConst = - typename std::conditional<std::is_const<From>::value, const To, To>::type; - -// Note: We're not qualifying this with absl:: because it doesn't compile under -// MSVC. -template <class T> -using SliceType = Span<T>; - -// This namespace contains no types. It prevents functions defined in it from -// being found by ADL. -namespace adl_barrier { - -template <class Needle, class... Ts> -constexpr size_t Find(Needle, Needle, Ts...) { - static_assert(!Contains<Needle, Ts...>(), "Duplicate element type"); - return 0; -} - -template <class Needle, class T, class... Ts> -constexpr size_t Find(Needle, T, Ts...) { - return adl_barrier::Find(Needle(), Ts()...) + 1; -} - -constexpr bool IsPow2(size_t n) { return !(n & (n - 1)); } - -// Returns `q * m` for the smallest `q` such that `q * m >= n`. -// Requires: `m` is a power of two. It's enforced by IsLegalElementType below. -constexpr size_t Align(size_t n, size_t m) { return (n + m - 1) & ~(m - 1); } - -constexpr size_t Min(size_t a, size_t b) { return b < a ? b : a; } - -constexpr size_t Max(size_t a) { return a; } - -template <class... Ts> -constexpr size_t Max(size_t a, size_t b, Ts... rest) { - return adl_barrier::Max(b < a ? a : b, rest...); -} - -template <class T> -std::string TypeName() { - std::string out; - int status = 0; - char* demangled = nullptr; -#ifdef ABSL_INTERNAL_HAS_CXA_DEMANGLE - demangled = abi::__cxa_demangle(typeid(T).name(), nullptr, nullptr, &status); -#endif - if (status == 0 && demangled != nullptr) { // Demangling succeeded. - absl::StrAppend(&out, "<", demangled, ">"); - free(demangled); - } else { -#if defined(__GXX_RTTI) || defined(_CPPRTTI) - absl::StrAppend(&out, "<", typeid(T).name(), ">"); -#endif - } - return out; -} - -} // namespace adl_barrier - -template <bool C> -using EnableIf = typename std::enable_if<C, int>::type; - -// Can `T` be a template argument of `Layout`? -template <class T> -using IsLegalElementType = std::integral_constant< - bool, !std::is_reference<T>::value && !std::is_volatile<T>::value && - !std::is_reference<typename Type<T>::type>::value && - !std::is_volatile<typename Type<T>::type>::value && - adl_barrier::IsPow2(AlignOf<T>::value)>; - -template <class Elements, class SizeSeq, class OffsetSeq> -class LayoutImpl; - -// Public base class of `Layout` and the result type of `Layout::Partial()`. -// -// `Elements...` contains all template arguments of `Layout` that created this -// instance. -// -// `SizeSeq...` is `[0, NumSizes)` where `NumSizes` is the number of arguments -// passed to `Layout::Partial()` or `Layout::Layout()`. -// -// `OffsetSeq...` is `[0, NumOffsets)` where `NumOffsets` is -// `Min(sizeof...(Elements), NumSizes + 1)` (the number of arrays for which we -// can compute offsets). -template <class... Elements, size_t... SizeSeq, size_t... OffsetSeq> -class LayoutImpl<std::tuple<Elements...>, absl::index_sequence<SizeSeq...>, - absl::index_sequence<OffsetSeq...>> { - private: - static_assert(sizeof...(Elements) > 0, "At least one field is required"); - static_assert(absl::conjunction<IsLegalElementType<Elements>...>::value, - "Invalid element type (see IsLegalElementType)"); - - enum { - NumTypes = sizeof...(Elements), - NumSizes = sizeof...(SizeSeq), - NumOffsets = sizeof...(OffsetSeq), - }; - - // These are guaranteed by `Layout`. - static_assert(NumOffsets == adl_barrier::Min(NumTypes, NumSizes + 1), - "Internal error"); - static_assert(NumTypes > 0, "Internal error"); - - // Returns the index of `T` in `Elements...`. Results in a compilation error - // if `Elements...` doesn't contain exactly one instance of `T`. - template <class T> - static constexpr size_t ElementIndex() { - static_assert(Contains<Type<T>, Type<typename Type<Elements>::type>...>(), - "Type not found"); - return adl_barrier::Find(Type<T>(), - Type<typename Type<Elements>::type>()...); - } - - template <size_t N> - using ElementAlignment = - AlignOf<typename std::tuple_element<N, std::tuple<Elements...>>::type>; - - public: - // Element types of all arrays packed in a tuple. - using ElementTypes = std::tuple<typename Type<Elements>::type...>; - - // Element type of the Nth array. - template <size_t N> - using ElementType = typename std::tuple_element<N, ElementTypes>::type; - - constexpr explicit LayoutImpl(IntToSize<SizeSeq>... sizes) - : size_{sizes...} {} - - // Alignment of the layout, equal to the strictest alignment of all elements. - // All pointers passed to the methods of layout must be aligned to this value. - static constexpr size_t Alignment() { - return adl_barrier::Max(AlignOf<Elements>::value...); - } - - // Offset in bytes of the Nth array. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // assert(x.Offset<0>() == 0); // The ints starts from 0. - // assert(x.Offset<1>() == 16); // The doubles starts from 16. - // - // Requires: `N <= NumSizes && N < sizeof...(Ts)`. - template <size_t N, EnableIf<N == 0> = 0> - constexpr size_t Offset() const { - return 0; - } - - template <size_t N, EnableIf<N != 0> = 0> - constexpr size_t Offset() const { - static_assert(N < NumOffsets, "Index out of bounds"); - return adl_barrier::Align( - Offset<N - 1>() + SizeOf<ElementType<N - 1>>::value * size_[N - 1], - ElementAlignment<N>::value); - } - - // Offset in bytes of the array with the specified element type. There must - // be exactly one such array and its zero-based index must be at most - // `NumSizes`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // assert(x.Offset<int>() == 0); // The ints starts from 0. - // assert(x.Offset<double>() == 16); // The doubles starts from 16. - template <class T> - constexpr size_t Offset() const { - return Offset<ElementIndex<T>()>(); - } - - // Offsets in bytes of all arrays for which the offsets are known. - constexpr std::array<size_t, NumOffsets> Offsets() const { - return {{Offset<OffsetSeq>()...}}; - } - - // The number of elements in the Nth array. This is the Nth argument of - // `Layout::Partial()` or `Layout::Layout()` (zero-based). - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // assert(x.Size<0>() == 3); - // assert(x.Size<1>() == 4); - // - // Requires: `N < NumSizes`. - template <size_t N> - constexpr size_t Size() const { - static_assert(N < NumSizes, "Index out of bounds"); - return size_[N]; - } - - // The number of elements in the array with the specified element type. - // There must be exactly one such array and its zero-based index must be - // at most `NumSizes`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // assert(x.Size<int>() == 3); - // assert(x.Size<double>() == 4); - template <class T> - constexpr size_t Size() const { - return Size<ElementIndex<T>()>(); - } - - // The number of elements of all arrays for which they are known. - constexpr std::array<size_t, NumSizes> Sizes() const { - return {{Size<SizeSeq>()...}}; - } - - // Pointer to the beginning of the Nth array. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; - // int* ints = x.Pointer<0>(p); - // double* doubles = x.Pointer<1>(p); - // - // Requires: `N <= NumSizes && N < sizeof...(Ts)`. - // Requires: `p` is aligned to `Alignment()`. - template <size_t N, class Char> - CopyConst<Char, ElementType<N>>* Pointer(Char* p) const { - using C = typename std::remove_const<Char>::type; - static_assert( - std::is_same<C, char>() || std::is_same<C, unsigned char>() || - std::is_same<C, signed char>(), - "The argument must be a pointer to [const] [signed|unsigned] char"); - constexpr size_t alignment = Alignment(); - (void)alignment; - assert(reinterpret_cast<uintptr_t>(p) % alignment == 0); - return reinterpret_cast<CopyConst<Char, ElementType<N>>*>(p + Offset<N>()); - } - - // Pointer to the beginning of the array with the specified element type. - // There must be exactly one such array and its zero-based index must be at - // most `NumSizes`. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; - // int* ints = x.Pointer<int>(p); - // double* doubles = x.Pointer<double>(p); - // - // Requires: `p` is aligned to `Alignment()`. - template <class T, class Char> - CopyConst<Char, T>* Pointer(Char* p) const { - return Pointer<ElementIndex<T>()>(p); - } - - // Pointers to all arrays for which pointers are known. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; - // - // int* ints; - // double* doubles; - // std::tie(ints, doubles) = x.Pointers(p); - // - // Requires: `p` is aligned to `Alignment()`. - // - // Note: We're not using ElementType alias here because it does not compile - // under MSVC. - template <class Char> - std::tuple<CopyConst< - Char, typename std::tuple_element<OffsetSeq, ElementTypes>::type>*...> - Pointers(Char* p) const { - return std::tuple<CopyConst<Char, ElementType<OffsetSeq>>*...>( - Pointer<OffsetSeq>(p)...); - } - - // The Nth array. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; - // Span<int> ints = x.Slice<0>(p); - // Span<double> doubles = x.Slice<1>(p); - // - // Requires: `N < NumSizes`. - // Requires: `p` is aligned to `Alignment()`. - template <size_t N, class Char> - SliceType<CopyConst<Char, ElementType<N>>> Slice(Char* p) const { - return SliceType<CopyConst<Char, ElementType<N>>>(Pointer<N>(p), Size<N>()); - } - - // The array with the specified element type. There must be exactly one - // such array and its zero-based index must be less than `NumSizes`. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; - // Span<int> ints = x.Slice<int>(p); - // Span<double> doubles = x.Slice<double>(p); - // - // Requires: `p` is aligned to `Alignment()`. - template <class T, class Char> - SliceType<CopyConst<Char, T>> Slice(Char* p) const { - return Slice<ElementIndex<T>()>(p); - } - - // All arrays with known sizes. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; - // - // Span<int> ints; - // Span<double> doubles; - // std::tie(ints, doubles) = x.Slices(p); - // - // Requires: `p` is aligned to `Alignment()`. - // - // Note: We're not using ElementType alias here because it does not compile - // under MSVC. - template <class Char> - std::tuple<SliceType<CopyConst< - Char, typename std::tuple_element<SizeSeq, ElementTypes>::type>>...> - Slices(Char* p) const { - // Workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63875 (fixed - // in 6.1). - (void)p; - return std::tuple<SliceType<CopyConst<Char, ElementType<SizeSeq>>>...>( - Slice<SizeSeq>(p)...); - } - - // The size of the allocation that fits all arrays. - // - // // int[3], 4 bytes of padding, double[4]. - // Layout<int, double> x(3, 4); - // unsigned char* p = new unsigned char[x.AllocSize()]; // 48 bytes - // - // Requires: `NumSizes == sizeof...(Ts)`. - constexpr size_t AllocSize() const { - static_assert(NumTypes == NumSizes, "You must specify sizes of all fields"); - return Offset<NumTypes - 1>() + - SizeOf<ElementType<NumTypes - 1>>::value * size_[NumTypes - 1]; - } - - // If built with --config=asan, poisons padding bytes (if any) in the - // allocation. The pointer must point to a memory block at least - // `AllocSize()` bytes in length. - // - // `Char` must be `[const] [signed|unsigned] char`. - // - // Requires: `p` is aligned to `Alignment()`. - template <class Char, size_t N = NumOffsets - 1, EnableIf<N == 0> = 0> - void PoisonPadding(const Char* p) const { - Pointer<0>(p); // verify the requirements on `Char` and `p` - } - - template <class Char, size_t N = NumOffsets - 1, EnableIf<N != 0> = 0> - void PoisonPadding(const Char* p) const { - static_assert(N < NumOffsets, "Index out of bounds"); - (void)p; -#ifdef ABSL_HAVE_ADDRESS_SANITIZER - PoisonPadding<Char, N - 1>(p); - // The `if` is an optimization. It doesn't affect the observable behaviour. - if (ElementAlignment<N - 1>::value % ElementAlignment<N>::value) { - size_t start = - Offset<N - 1>() + SizeOf<ElementType<N - 1>>::value * size_[N - 1]; - ASAN_POISON_MEMORY_REGION(p + start, Offset<N>() - start); - } -#endif - } - - // Human-readable description of the memory layout. Useful for debugging. - // Slow. - // - // // char[5], 3 bytes of padding, int[3], 4 bytes of padding, followed - // // by an unknown number of doubles. - // auto x = Layout<char, int, double>::Partial(5, 3); - // assert(x.DebugString() == - // "@0<char>(1)[5]; @8<int>(4)[3]; @24<double>(8)"); - // - // Each field is in the following format: @offset<type>(sizeof)[size] (<type> - // may be missing depending on the target platform). For example, - // @8<int>(4)[3] means that at offset 8 we have an array of ints, where each - // int is 4 bytes, and we have 3 of those ints. The size of the last field may - // be missing (as in the example above). Only fields with known offsets are - // described. Type names may differ across platforms: one compiler might - // produce "unsigned*" where another produces "unsigned int *". - std::string DebugString() const { - const auto offsets = Offsets(); - const size_t sizes[] = {SizeOf<ElementType<OffsetSeq>>::value...}; - const std::string types[] = { - adl_barrier::TypeName<ElementType<OffsetSeq>>()...}; - std::string res = absl::StrCat("@0", types[0], "(", sizes[0], ")"); - for (size_t i = 0; i != NumOffsets - 1; ++i) { - absl::StrAppend(&res, "[", size_[i], "]; @", offsets[i + 1], types[i + 1], - "(", sizes[i + 1], ")"); - } - // NumSizes is a constant that may be zero. Some compilers cannot see that - // inside the if statement "size_[NumSizes - 1]" must be valid. - int last = static_cast<int>(NumSizes) - 1; - if (NumTypes == NumSizes && last >= 0) { - absl::StrAppend(&res, "[", size_[last], "]"); - } - return res; - } - - private: - // Arguments of `Layout::Partial()` or `Layout::Layout()`. - size_t size_[NumSizes > 0 ? NumSizes : 1]; -}; - -template <size_t NumSizes, class... Ts> -using LayoutType = LayoutImpl< - std::tuple<Ts...>, absl::make_index_sequence<NumSizes>, - absl::make_index_sequence<adl_barrier::Min(sizeof...(Ts), NumSizes + 1)>>; - -} // namespace internal_layout - -// Descriptor of arrays of various types and sizes laid out in memory one after -// another. See the top of the file for documentation. -// -// Check out the public API of internal_layout::LayoutImpl above. The type is -// internal to the library but its methods are public, and they are inherited -// by `Layout`. -template <class... Ts> -class Layout : public internal_layout::LayoutType<sizeof...(Ts), Ts...> { - public: - static_assert(sizeof...(Ts) > 0, "At least one field is required"); - static_assert( - absl::conjunction<internal_layout::IsLegalElementType<Ts>...>::value, - "Invalid element type (see IsLegalElementType)"); - - // The result type of `Partial()` with `NumSizes` arguments. - template <size_t NumSizes> - using PartialType = internal_layout::LayoutType<NumSizes, Ts...>; - - // `Layout` knows the element types of the arrays we want to lay out in - // memory but not the number of elements in each array. - // `Partial(size1, ..., sizeN)` allows us to specify the latter. The - // resulting immutable object can be used to obtain pointers to the - // individual arrays. - // - // It's allowed to pass fewer array sizes than the number of arrays. E.g., - // if all you need is to the offset of the second array, you only need to - // pass one argument -- the number of elements in the first array. - // - // // int[3] followed by 4 bytes of padding and an unknown number of - // // doubles. - // auto x = Layout<int, double>::Partial(3); - // // doubles start at byte 16. - // assert(x.Offset<1>() == 16); - // - // If you know the number of elements in all arrays, you can still call - // `Partial()` but it's more convenient to use the constructor of `Layout`. - // - // Layout<int, double> x(3, 5); - // - // Note: The sizes of the arrays must be specified in number of elements, - // not in bytes. - // - // Requires: `sizeof...(Sizes) <= sizeof...(Ts)`. - // Requires: all arguments are convertible to `size_t`. - template <class... Sizes> - static constexpr PartialType<sizeof...(Sizes)> Partial(Sizes&&... sizes) { - static_assert(sizeof...(Sizes) <= sizeof...(Ts), ""); - return PartialType<sizeof...(Sizes)>(absl::forward<Sizes>(sizes)...); - } - - // Creates a layout with the sizes of all arrays specified. If you know - // only the sizes of the first N arrays (where N can be zero), you can use - // `Partial()` defined above. The constructor is essentially equivalent to - // calling `Partial()` and passing in all array sizes; the constructor is - // provided as a convenient abbreviation. - // - // Note: The sizes of the arrays must be specified in number of elements, - // not in bytes. - constexpr explicit Layout(internal_layout::TypeToSize<Ts>... sizes) - : internal_layout::LayoutType<sizeof...(Ts), Ts...>(sizes...) {} -}; - -} // namespace container_internal -ABSL_NAMESPACE_END -} // namespace absl - -#endif // ABSL_CONTAINER_INTERNAL_LAYOUT_H_ diff --git a/contrib/restricted/abseil-cpp/absl/container/internal/raw_hash_map.h b/contrib/restricted/abseil-cpp/absl/container/internal/raw_hash_map.h index 2d5a8710ad..97182bc7bd 100644 --- a/contrib/restricted/abseil-cpp/absl/container/internal/raw_hash_map.h +++ b/contrib/restricted/abseil-cpp/absl/container/internal/raw_hash_map.h @@ -19,6 +19,8 @@ #include <type_traits> #include <utility> +#include "absl/base/attributes.h" +#include "absl/base/config.h" #include "absl/base/internal/throw_delegate.h" #include "absl/container/internal/container_memory.h" #include "absl/container/internal/raw_hash_set.h" // IWYU pragma: export @@ -175,13 +177,20 @@ class raw_hash_map : public raw_hash_set<Policy, Hash, Eq, Alloc> { template <class K = key_type, class P = Policy, K* = nullptr> MappedReference<P> operator[](key_arg<K>&& key) ABSL_ATTRIBUTE_LIFETIME_BOUND { - return Policy::value(&*try_emplace(std::forward<K>(key)).first); + // It is safe to use unchecked_deref here because try_emplace + // will always return an iterator pointing to a valid item in the table, + // since it inserts if nothing is found for the given key. + return Policy::value( + &this->unchecked_deref(try_emplace(std::forward<K>(key)).first)); } template <class K = key_type, class P = Policy> MappedReference<P> operator[](const key_arg<K>& key) ABSL_ATTRIBUTE_LIFETIME_BOUND { - return Policy::value(&*try_emplace(key).first); + // It is safe to use unchecked_deref here because try_emplace + // will always return an iterator pointing to a valid item in the table, + // since it inserts if nothing is found for the given key. + return Policy::value(&this->unchecked_deref(try_emplace(key).first)); } private: diff --git a/contrib/restricted/abseil-cpp/absl/container/internal/raw_hash_set.cc b/contrib/restricted/abseil-cpp/absl/container/internal/raw_hash_set.cc index 2ff95b6182..9f8ea51987 100644 --- a/contrib/restricted/abseil-cpp/absl/container/internal/raw_hash_set.cc +++ b/contrib/restricted/abseil-cpp/absl/container/internal/raw_hash_set.cc @@ -17,11 +17,13 @@ #include <atomic> #include <cassert> #include <cstddef> +#include <cstdint> #include <cstring> #include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/dynamic_annotations.h" +#include "absl/container/internal/container_memory.h" #include "absl/hash/hash.h" namespace absl { @@ -67,6 +69,16 @@ inline size_t RandomSeed() { return value ^ static_cast<size_t>(reinterpret_cast<uintptr_t>(&counter)); } +bool ShouldRehashForBugDetection(const ctrl_t* ctrl, size_t capacity) { + // Note: we can't use the abseil-random library because abseil-random + // depends on swisstable. We want to return true with probability + // `min(1, RehashProbabilityConstant() / capacity())`. In order to do this, + // we probe based on a random hash and see if the offset is less than + // RehashProbabilityConstant(). + return probe(ctrl, capacity, absl::HashOf(RandomSeed())).offset() < + RehashProbabilityConstant(); +} + } // namespace GenerationType* EmptyGeneration() { @@ -84,13 +96,12 @@ bool CommonFieldsGenerationInfoEnabled:: size_t capacity) const { if (reserved_growth_ == kReservedGrowthJustRanOut) return true; if (reserved_growth_ > 0) return false; - // Note: we can't use the abseil-random library because abseil-random - // depends on swisstable. We want to return true with probability - // `min(1, RehashProbabilityConstant() / capacity())`. In order to do this, - // we probe based on a random hash and see if the offset is less than - // RehashProbabilityConstant(). - return probe(ctrl, capacity, absl::HashOf(RandomSeed())).offset() < - RehashProbabilityConstant(); + return ShouldRehashForBugDetection(ctrl, capacity); +} + +bool CommonFieldsGenerationInfoEnabled::should_rehash_for_bug_detection_on_move( + const ctrl_t* ctrl, size_t capacity) const { + return ShouldRehashForBugDetection(ctrl, capacity); } bool ShouldInsertBackwards(size_t hash, const ctrl_t* ctrl) { @@ -117,14 +128,6 @@ FindInfo find_first_non_full_outofline(const CommonFields& common, return find_first_non_full(common, hash); } -// Returns the address of the ith slot in slots where each slot occupies -// slot_size. -static inline void* SlotAddress(void* slot_array, size_t slot, - size_t slot_size) { - return reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(slot_array) + - (slot * slot_size)); -} - // Returns the address of the slot just after slot assuming each slot has the // specified size. static inline void* NextSlot(void* slot, size_t slot_size) { @@ -218,26 +221,35 @@ void DropDeletesWithoutResize(CommonFields& common, common.infoz().RecordRehash(total_probe_length); } -void EraseMetaOnly(CommonFields& c, ctrl_t* it, size_t slot_size) { - assert(IsFull(*it) && "erasing a dangling iterator"); - c.set_size(c.size() - 1); - const auto index = static_cast<size_t>(it - c.control()); +static bool WasNeverFull(CommonFields& c, size_t index) { + if (is_single_group(c.capacity())) { + return true; + } const size_t index_before = (index - Group::kWidth) & c.capacity(); - const auto empty_after = Group(it).MaskEmpty(); + const auto empty_after = Group(c.control() + index).MaskEmpty(); const auto empty_before = Group(c.control() + index_before).MaskEmpty(); // We count how many consecutive non empties we have to the right and to the // left of `it`. If the sum is >= kWidth then there is at least one probe // window that might have seen a full group. - bool was_never_full = empty_before && empty_after && - static_cast<size_t>(empty_after.TrailingZeros()) + - empty_before.LeadingZeros() < - Group::kWidth; - - SetCtrl(c, index, was_never_full ? ctrl_t::kEmpty : ctrl_t::kDeleted, - slot_size); - c.set_growth_left(c.growth_left() + (was_never_full ? 1 : 0)); + return empty_before && empty_after && + static_cast<size_t>(empty_after.TrailingZeros()) + + empty_before.LeadingZeros() < + Group::kWidth; +} + +void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size) { + assert(IsFull(c.control()[index]) && "erasing a dangling iterator"); + c.decrement_size(); c.infoz().RecordErase(); + + if (WasNeverFull(c, index)) { + SetCtrl(c, index, ctrl_t::kEmpty, slot_size); + c.set_growth_left(c.growth_left() + 1); + return; + } + + SetCtrl(c, index, ctrl_t::kDeleted, slot_size); } void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy, @@ -245,19 +257,124 @@ void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy, c.set_size(0); if (reuse) { ResetCtrl(c, policy.slot_size); + ResetGrowthLeft(c); c.infoz().RecordStorageChanged(0, c.capacity()); } else { + // We need to record infoz before calling dealloc, which will unregister + // infoz. + c.infoz().RecordClearedReservation(); + c.infoz().RecordStorageChanged(0, 0); (*policy.dealloc)(c, policy); c.set_control(EmptyGroup()); c.set_generation_ptr(EmptyGeneration()); c.set_slots(nullptr); c.set_capacity(0); - c.infoz().RecordClearedReservation(); - assert(c.size() == 0); - c.infoz().RecordStorageChanged(0, 0); } } +void HashSetResizeHelper::GrowIntoSingleGroupShuffleControlBytes( + ctrl_t* new_ctrl, size_t new_capacity) const { + assert(is_single_group(new_capacity)); + constexpr size_t kHalfWidth = Group::kWidth / 2; + assert(old_capacity_ < kHalfWidth); + + const size_t half_old_capacity = old_capacity_ / 2; + + // NOTE: operations are done with compile time known size = kHalfWidth. + // Compiler optimizes that into single ASM operation. + + // Copy second half of bytes to the beginning. + // We potentially copy more bytes in order to have compile time known size. + // Mirrored bytes from the old_ctrl_ will also be copied. + // In case of old_capacity_ == 3, we will copy 1st element twice. + // Examples: + // old_ctrl = 0S0EEEEEEE... + // new_ctrl = S0EEEEEEEE... + // + // old_ctrl = 01S01EEEEE... + // new_ctrl = 1S01EEEEEE... + // + // old_ctrl = 0123456S0123456EE... + // new_ctrl = 456S0123?????????... + std::memcpy(new_ctrl, old_ctrl_ + half_old_capacity + 1, kHalfWidth); + // Clean up copied kSentinel from old_ctrl. + new_ctrl[half_old_capacity] = ctrl_t::kEmpty; + + // Clean up damaged or uninitialized bytes. + + // Clean bytes after the intended size of the copy. + // Example: + // new_ctrl = 1E01EEEEEEE???? + // *new_ctrl= 1E0EEEEEEEE???? + // position / + std::memset(new_ctrl + old_capacity_ + 1, static_cast<int8_t>(ctrl_t::kEmpty), + kHalfWidth); + // Clean non-mirrored bytes that are not initialized. + // For small old_capacity that may be inside of mirrored bytes zone. + // Examples: + // new_ctrl = 1E0EEEEEEEE??????????.... + // *new_ctrl= 1E0EEEEEEEEEEEEE?????.... + // position / + // + // new_ctrl = 456E0123???????????... + // *new_ctrl= 456E0123EEEEEEEE???... + // position / + std::memset(new_ctrl + kHalfWidth, static_cast<int8_t>(ctrl_t::kEmpty), + kHalfWidth); + // Clean last mirrored bytes that are not initialized + // and will not be overwritten by mirroring. + // Examples: + // new_ctrl = 1E0EEEEEEEEEEEEE???????? + // *new_ctrl= 1E0EEEEEEEEEEEEEEEEEEEEE + // position S / + // + // new_ctrl = 456E0123EEEEEEEE??????????????? + // *new_ctrl= 456E0123EEEEEEEE???????EEEEEEEE + // position S / + std::memset(new_ctrl + new_capacity + kHalfWidth, + static_cast<int8_t>(ctrl_t::kEmpty), kHalfWidth); + + // Create mirrored bytes. old_capacity_ < kHalfWidth + // Example: + // new_ctrl = 456E0123EEEEEEEE???????EEEEEEEE + // *new_ctrl= 456E0123EEEEEEEE456E0123EEEEEEE + // position S/ + ctrl_t g[kHalfWidth]; + std::memcpy(g, new_ctrl, kHalfWidth); + std::memcpy(new_ctrl + new_capacity + 1, g, kHalfWidth); + + // Finally set sentinel to its place. + new_ctrl[new_capacity] = ctrl_t::kSentinel; +} + +void HashSetResizeHelper::GrowIntoSingleGroupShuffleTransferableSlots( + void* old_slots, void* new_slots, size_t slot_size) const { + assert(old_capacity_ > 0); + const size_t half_old_capacity = old_capacity_ / 2; + + SanitizerUnpoisonMemoryRegion(old_slots, slot_size * old_capacity_); + std::memcpy(new_slots, + SlotAddress(old_slots, half_old_capacity + 1, slot_size), + slot_size * half_old_capacity); + std::memcpy(SlotAddress(new_slots, half_old_capacity + 1, slot_size), + old_slots, slot_size * (half_old_capacity + 1)); +} + +void HashSetResizeHelper::GrowSizeIntoSingleGroupTransferable( + CommonFields& c, void* old_slots, size_t slot_size) { + assert(old_capacity_ < Group::kWidth / 2); + assert(is_single_group(c.capacity())); + assert(IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity())); + + GrowIntoSingleGroupShuffleControlBytes(c.control(), c.capacity()); + GrowIntoSingleGroupShuffleTransferableSlots(old_slots, c.slot_array(), + slot_size); + + // We poison since GrowIntoSingleGroupShuffleTransferableSlots + // may leave empty slots unpoisoned. + PoisonSingleGroupEmptySlots(c, slot_size); +} + } // namespace container_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/contrib/restricted/abseil-cpp/absl/container/internal/raw_hash_set.h b/contrib/restricted/abseil-cpp/absl/container/internal/raw_hash_set.h index 5f89d8efee..3518bc3404 100644 --- a/contrib/restricted/abseil-cpp/absl/container/internal/raw_hash_set.h +++ b/contrib/restricted/abseil-cpp/absl/container/internal/raw_hash_set.h @@ -62,6 +62,9 @@ // pseudo-struct: // // struct BackingArray { +// // Sampling handler. This field isn't present when the sampling is +// // disabled or this allocation hasn't been selected for sampling. +// HashtablezInfoHandle infoz_; // // The number of elements we can insert before growing the capacity. // size_t growth_left; // // Control bytes for the "real" slots. @@ -175,25 +178,29 @@ #define ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ #include <algorithm> +#include <cassert> #include <cmath> #include <cstddef> #include <cstdint> #include <cstring> +#include <initializer_list> #include <iterator> #include <limits> #include <memory> -#include <string> #include <tuple> #include <type_traits> #include <utility> +#include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/internal/endian.h" #include "absl/base/internal/raw_logging.h" +#include "absl/base/macros.h" #include "absl/base/optimization.h" +#include "absl/base/options.h" #include "absl/base/port.h" #include "absl/base/prefetch.h" -#include "absl/container/internal/common.h" +#include "absl/container/internal/common.h" // IWYU pragma: export // for node_handle #include "absl/container/internal/compressed_tuple.h" #include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_policy_traits.h" @@ -227,6 +234,7 @@ namespace container_internal { #ifdef ABSL_SWISSTABLE_ENABLE_GENERATIONS #error ABSL_SWISSTABLE_ENABLE_GENERATIONS cannot be directly set #elif defined(ABSL_HAVE_ADDRESS_SANITIZER) || \ + defined(ABSL_HAVE_HWADDRESS_SANITIZER) || \ defined(ABSL_HAVE_MEMORY_SANITIZER) // When compiled in sanitizer mode, we add generation integers to the backing // array and iterators. In the backing array, we store the generation between @@ -262,8 +270,21 @@ void SwapAlloc(AllocType& lhs, AllocType& rhs, swap(lhs, rhs); } template <typename AllocType> -void SwapAlloc(AllocType& /*lhs*/, AllocType& /*rhs*/, - std::false_type /* propagate_on_container_swap */) {} +void SwapAlloc(AllocType& lhs, AllocType& rhs, + std::false_type /* propagate_on_container_swap */) { + (void)lhs; + (void)rhs; + assert(lhs == rhs && + "It's UB to call swap with unequal non-propagating allocators."); +} + +template <typename AllocType> +void CopyAlloc(AllocType& lhs, AllocType& rhs, + std::true_type /* propagate_alloc */) { + lhs = rhs; +} +template <typename AllocType> +void CopyAlloc(AllocType&, AllocType&, std::false_type /* propagate_alloc */) {} // The state for a probe sequence. // @@ -361,7 +382,7 @@ uint32_t TrailingZeros(T x) { // width of an abstract bit in the representation. // This mask provides operations for any number of real bits set in an abstract // bit. To add iteration on top of that, implementation must guarantee no more -// than one real bit is set in an abstract bit. +// than the most significant real bit is set in a set abstract bit. template <class T, int SignificantBits, int Shift = 0> class NonIterableBitMask { public: @@ -388,7 +409,9 @@ class NonIterableBitMask { uint32_t LeadingZeros() const { constexpr int total_significant_bits = SignificantBits << Shift; constexpr int extra_bits = sizeof(T) * 8 - total_significant_bits; - return static_cast<uint32_t>(countl_zero(mask_ << extra_bits)) >> Shift; + return static_cast<uint32_t>( + countl_zero(static_cast<T>(mask_ << extra_bits))) >> + Shift; } T mask_; @@ -418,6 +441,10 @@ class BitMask : public NonIterableBitMask<T, SignificantBits, Shift> { using const_iterator = BitMask; BitMask& operator++() { + if (Shift == 3) { + constexpr uint64_t msbs = 0x8080808080808080ULL; + this->mask_ &= msbs; + } this->mask_ &= (this->mask_ - 1); return *this; } @@ -590,29 +617,39 @@ struct GroupSse2Impl { } // Returns a bitmask representing the positions of slots that match hash. - BitMask<uint32_t, kWidth> Match(h2_t hash) const { + BitMask<uint16_t, kWidth> Match(h2_t hash) const { auto match = _mm_set1_epi8(static_cast<char>(hash)); - return BitMask<uint32_t, kWidth>( - static_cast<uint32_t>(_mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl)))); + BitMask<uint16_t, kWidth> result = BitMask<uint16_t, kWidth>(0); + result = BitMask<uint16_t, kWidth>( + static_cast<uint16_t>(_mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl)))); + return result; } // Returns a bitmask representing the positions of empty slots. - NonIterableBitMask<uint32_t, kWidth> MaskEmpty() const { + NonIterableBitMask<uint16_t, kWidth> MaskEmpty() const { #ifdef ABSL_INTERNAL_HAVE_SSSE3 // This only works because ctrl_t::kEmpty is -128. - return NonIterableBitMask<uint32_t, kWidth>( - static_cast<uint32_t>(_mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl)))); + return NonIterableBitMask<uint16_t, kWidth>( + static_cast<uint16_t>(_mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl)))); #else auto match = _mm_set1_epi8(static_cast<char>(ctrl_t::kEmpty)); - return NonIterableBitMask<uint32_t, kWidth>( - static_cast<uint32_t>(_mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl)))); + return NonIterableBitMask<uint16_t, kWidth>( + static_cast<uint16_t>(_mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl)))); #endif } + // Returns a bitmask representing the positions of full slots. + // Note: for `is_small()` tables group may contain the "same" slot twice: + // original and mirrored. + BitMask<uint16_t, kWidth> MaskFull() const { + return BitMask<uint16_t, kWidth>( + static_cast<uint16_t>(_mm_movemask_epi8(ctrl) ^ 0xffff)); + } + // Returns a bitmask representing the positions of empty or deleted slots. - NonIterableBitMask<uint32_t, kWidth> MaskEmptyOrDeleted() const { + NonIterableBitMask<uint16_t, kWidth> MaskEmptyOrDeleted() const { auto special = _mm_set1_epi8(static_cast<char>(ctrl_t::kSentinel)); - return NonIterableBitMask<uint32_t, kWidth>(static_cast<uint32_t>( + return NonIterableBitMask<uint16_t, kWidth>(static_cast<uint16_t>( _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)))); } @@ -651,9 +688,8 @@ struct GroupAArch64Impl { BitMask<uint64_t, kWidth, 3> Match(h2_t hash) const { uint8x8_t dup = vdup_n_u8(hash); auto mask = vceq_u8(ctrl, dup); - constexpr uint64_t msbs = 0x8080808080808080ULL; return BitMask<uint64_t, kWidth, 3>( - vget_lane_u64(vreinterpret_u64_u8(mask), 0) & msbs); + vget_lane_u64(vreinterpret_u64_u8(mask), 0)); } NonIterableBitMask<uint64_t, kWidth, 3> MaskEmpty() const { @@ -665,6 +701,17 @@ struct GroupAArch64Impl { return NonIterableBitMask<uint64_t, kWidth, 3>(mask); } + // Returns a bitmask representing the positions of full slots. + // Note: for `is_small()` tables group may contain the "same" slot twice: + // original and mirrored. + BitMask<uint64_t, kWidth, 3> MaskFull() const { + uint64_t mask = vget_lane_u64( + vreinterpret_u64_u8(vcge_s8(vreinterpret_s8_u8(ctrl), + vdup_n_s8(static_cast<int8_t>(0)))), + 0); + return BitMask<uint64_t, kWidth, 3>(mask); + } + NonIterableBitMask<uint64_t, kWidth, 3> MaskEmptyOrDeleted() const { uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(vcgt_s8( @@ -729,13 +776,21 @@ struct GroupPortableImpl { NonIterableBitMask<uint64_t, kWidth, 3> MaskEmpty() const { constexpr uint64_t msbs = 0x8080808080808080ULL; - return NonIterableBitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 6)) & + return NonIterableBitMask<uint64_t, kWidth, 3>((ctrl & ~(ctrl << 6)) & msbs); } + // Returns a bitmask representing the positions of full slots. + // Note: for `is_small()` tables group may contain the "same" slot twice: + // original and mirrored. + BitMask<uint64_t, kWidth, 3> MaskFull() const { + constexpr uint64_t msbs = 0x8080808080808080ULL; + return BitMask<uint64_t, kWidth, 3>((ctrl ^ msbs) & msbs); + } + NonIterableBitMask<uint64_t, kWidth, 3> MaskEmptyOrDeleted() const { constexpr uint64_t msbs = 0x8080808080808080ULL; - return NonIterableBitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 7)) & + return NonIterableBitMask<uint64_t, kWidth, 3>((ctrl & ~(ctrl << 7)) & msbs); } @@ -760,10 +815,21 @@ struct GroupPortableImpl { #ifdef ABSL_INTERNAL_HAVE_SSE2 using Group = GroupSse2Impl; +using GroupEmptyOrDeleted = GroupSse2Impl; #elif defined(ABSL_INTERNAL_HAVE_ARM_NEON) && defined(ABSL_IS_LITTLE_ENDIAN) using Group = GroupAArch64Impl; +// For Aarch64, we use the portable implementation for counting and masking +// empty or deleted group elements. This is to avoid the latency of moving +// between data GPRs and Neon registers when it does not provide a benefit. +// Using Neon is profitable when we call Match(), but is not when we don't, +// which is the case when we do *EmptyOrDeleted operations. It is difficult to +// make a similar approach beneficial on other architectures such as x86 since +// they have much lower GPR <-> vector register transfer latency and 16-wide +// Groups. +using GroupEmptyOrDeleted = GroupPortableImpl; #else using Group = GroupPortableImpl; +using GroupEmptyOrDeleted = GroupPortableImpl; #endif // When there is an insertion with no reserved growth, we rehash with @@ -802,15 +868,19 @@ class CommonFieldsGenerationInfoEnabled { // whenever reserved_growth_ is zero. bool should_rehash_for_bug_detection_on_insert(const ctrl_t* ctrl, size_t capacity) const; + // Similar to above, except that we don't depend on reserved_growth_. + bool should_rehash_for_bug_detection_on_move(const ctrl_t* ctrl, + size_t capacity) const; void maybe_increment_generation_on_insert() { if (reserved_growth_ == kReservedGrowthJustRanOut) reserved_growth_ = 0; if (reserved_growth_ > 0) { if (--reserved_growth_ == 0) reserved_growth_ = kReservedGrowthJustRanOut; } else { - *generation_ = NextGeneration(*generation_); + increment_generation(); } } + void increment_generation() { *generation_ = NextGeneration(*generation_); } void reset_reserved_growth(size_t reservation, size_t size) { reserved_growth_ = reservation - size; } @@ -856,7 +926,11 @@ class CommonFieldsGenerationInfoDisabled { bool should_rehash_for_bug_detection_on_insert(const ctrl_t*, size_t) const { return false; } + bool should_rehash_for_bug_detection_on_move(const ctrl_t*, size_t) const { + return false; + } void maybe_increment_generation_on_insert() {} + void increment_generation() {} void reset_reserved_growth(size_t, size_t) {} size_t reserved_growth() const { return 0; } void set_reserved_growth(size_t) {} @@ -909,9 +983,11 @@ using HashSetIteratorGenerationInfo = HashSetIteratorGenerationInfoDisabled; // A valid capacity is a non-zero integer `2^m - 1`. inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; } -// Computes the offset from the start of the backing allocation of the control -// bytes. growth_left is stored at the beginning of the backing array. -inline size_t ControlOffset() { return sizeof(size_t); } +// Computes the offset from the start of the backing allocation of control. +// infoz and growth_left are stored at the beginning of the backing array. +inline size_t ControlOffset(bool has_infoz) { + return (has_infoz ? sizeof(HashtablezInfoHandle) : 0) + sizeof(size_t); +} // Returns the number of "cloned control bytes". // @@ -922,24 +998,26 @@ constexpr size_t NumClonedBytes() { return Group::kWidth - 1; } // Given the capacity of a table, computes the offset (from the start of the // backing allocation) of the generation counter (if it exists). -inline size_t GenerationOffset(size_t capacity) { +inline size_t GenerationOffset(size_t capacity, bool has_infoz) { assert(IsValidCapacity(capacity)); const size_t num_control_bytes = capacity + 1 + NumClonedBytes(); - return ControlOffset() + num_control_bytes; + return ControlOffset(has_infoz) + num_control_bytes; } // Given the capacity of a table, computes the offset (from the start of the // backing allocation) at which the slots begin. -inline size_t SlotOffset(size_t capacity, size_t slot_align) { +inline size_t SlotOffset(size_t capacity, size_t slot_align, bool has_infoz) { assert(IsValidCapacity(capacity)); - return (GenerationOffset(capacity) + NumGenerationBytes() + slot_align - 1) & + return (GenerationOffset(capacity, has_infoz) + NumGenerationBytes() + + slot_align - 1) & (~slot_align + 1); } // Given the capacity of a table, computes the total size of the backing // array. -inline size_t AllocSize(size_t capacity, size_t slot_size, size_t slot_align) { - return SlotOffset(capacity, slot_align) + capacity * slot_size; +inline size_t AllocSize(size_t capacity, size_t slot_size, size_t slot_align, + bool has_infoz) { + return SlotOffset(capacity, slot_align, has_infoz) + capacity * slot_size; } // CommonFields hold the fields in raw_hash_set that do not depend @@ -954,28 +1032,15 @@ class CommonFields : public CommonFieldsGenerationInfo { CommonFields& operator=(const CommonFields&) = delete; // Movable - CommonFields(CommonFields&& that) - : CommonFieldsGenerationInfo( - std::move(static_cast<CommonFieldsGenerationInfo&&>(that))), - // Explicitly copying fields into "this" and then resetting "that" - // fields generates less code then calling absl::exchange per field. - control_(that.control()), - slots_(that.slot_array()), - capacity_(that.capacity()), - compressed_tuple_(that.size(), std::move(that.infoz())) { - that.set_control(EmptyGroup()); - that.set_slots(nullptr); - that.set_capacity(0); - that.set_size(0); - } + CommonFields(CommonFields&& that) = default; CommonFields& operator=(CommonFields&&) = default; ctrl_t* control() const { return control_; } void set_control(ctrl_t* c) { control_ = c; } void* backing_array_start() const { - // growth_left is stored before control bytes. + // growth_left (and maybe infoz) is stored before control bytes. assert(reinterpret_cast<uintptr_t>(control()) % alignof(size_t) == 0); - return control() - sizeof(size_t); + return control() - ControlOffset(has_infoz()); } // Note: we can't use slots() because Qt defines "slots" as a macro. @@ -983,8 +1048,18 @@ class CommonFields : public CommonFieldsGenerationInfo { void set_slots(void* s) { slots_ = s; } // The number of filled slots. - size_t size() const { return compressed_tuple_.template get<0>(); } - void set_size(size_t s) { compressed_tuple_.template get<0>() = s; } + size_t size() const { return size_ >> HasInfozShift(); } + void set_size(size_t s) { + size_ = (s << HasInfozShift()) | (size_ & HasInfozMask()); + } + void increment_size() { + assert(size() < capacity()); + size_ += size_t{1} << HasInfozShift(); + } + void decrement_size() { + assert(size() > 0); + size_ -= size_t{1} << HasInfozShift(); + } // The total number of available slots. size_t capacity() const { return capacity_; } @@ -996,28 +1071,52 @@ class CommonFields : public CommonFieldsGenerationInfo { // The number of slots we can still fill without needing to rehash. // This is stored in the heap allocation before the control bytes. size_t growth_left() const { - return *reinterpret_cast<size_t*>(backing_array_start()); + const size_t* gl_ptr = reinterpret_cast<size_t*>(control()) - 1; + assert(reinterpret_cast<uintptr_t>(gl_ptr) % alignof(size_t) == 0); + return *gl_ptr; } void set_growth_left(size_t gl) { - *reinterpret_cast<size_t*>(backing_array_start()) = gl; + size_t* gl_ptr = reinterpret_cast<size_t*>(control()) - 1; + assert(reinterpret_cast<uintptr_t>(gl_ptr) % alignof(size_t) == 0); + *gl_ptr = gl; } - HashtablezInfoHandle& infoz() { return compressed_tuple_.template get<1>(); } - const HashtablezInfoHandle& infoz() const { - return compressed_tuple_.template get<1>(); + bool has_infoz() const { + return ABSL_PREDICT_FALSE((size_ & HasInfozMask()) != 0); + } + void set_has_infoz(bool has_infoz) { + size_ = (size() << HasInfozShift()) | static_cast<size_t>(has_infoz); + } + + HashtablezInfoHandle infoz() { + return has_infoz() + ? *reinterpret_cast<HashtablezInfoHandle*>(backing_array_start()) + : HashtablezInfoHandle(); + } + void set_infoz(HashtablezInfoHandle infoz) { + assert(has_infoz()); + *reinterpret_cast<HashtablezInfoHandle*>(backing_array_start()) = infoz; } bool should_rehash_for_bug_detection_on_insert() const { return CommonFieldsGenerationInfo:: should_rehash_for_bug_detection_on_insert(control(), capacity()); } + bool should_rehash_for_bug_detection_on_move() const { + return CommonFieldsGenerationInfo:: + should_rehash_for_bug_detection_on_move(control(), capacity()); + } + void maybe_increment_generation_on_move() { + if (capacity() == 0) return; + increment_generation(); + } void reset_reserved_growth(size_t reservation) { CommonFieldsGenerationInfo::reset_reserved_growth(reservation, size()); } // The size of the backing array allocation. size_t alloc_size(size_t slot_size, size_t slot_align) const { - return AllocSize(capacity(), slot_size, slot_align); + return AllocSize(capacity(), slot_size, slot_align, has_infoz()); } // Returns the number of control bytes set to kDeleted. For testing only. @@ -1027,9 +1126,14 @@ class CommonFields : public CommonFieldsGenerationInfo { } private: - // TODO(b/259599413): Investigate removing some of these fields: + // We store the has_infoz bit in the lowest bit of size_. + static constexpr size_t HasInfozShift() { return 1; } + static constexpr size_t HasInfozMask() { + return (size_t{1} << HasInfozShift()) - 1; + } + + // TODO(b/182800944): Investigate removing some of these fields: // - control/slots can be derived from each other - // - we can use 6 bits for capacity since it's always a power of two minus 1 // The control bytes (and, also, a pointer near to the base of the backing // array). @@ -1044,12 +1148,16 @@ class CommonFields : public CommonFieldsGenerationInfo { // `control`. May be null for empty tables. void* slots_ = nullptr; + // The number of slots in the backing array. This is always 2^N-1 for an + // integer N. NOTE: we tried experimenting with compressing the capacity and + // storing it together with size_: (a) using 6 bits to store the corresponding + // power (N in 2^N-1), and (b) storing 2^N as the most significant bit of + // size_ and storing size in the low bits. Both of these experiments were + // regressions, presumably because we need capacity to do find operations. size_t capacity_ = 0; - // Bundle together size and HashtablezInfoHandle to ensure EBO for - // HashtablezInfoHandle when sampling is turned off. - absl::container_internal::CompressedTuple<size_t, HashtablezInfoHandle> - compressed_tuple_{0u, HashtablezInfoHandle{}}; + // The size and also has one bit that stores whether we have infoz. + size_t size_ = 0; }; template <class Policy, class Hash, class Eq, class Alloc> @@ -1139,35 +1247,39 @@ inline void AssertIsFull(const ctrl_t* ctrl, GenerationType generation, const GenerationType* generation_ptr, const char* operation) { if (!SwisstableDebugEnabled()) return; - if (ctrl == nullptr) { - ABSL_INTERNAL_LOG(FATAL, - std::string(operation) + " called on end() iterator."); - } - if (ctrl == EmptyGroup()) { - ABSL_INTERNAL_LOG(FATAL, std::string(operation) + - " called on default-constructed iterator."); + // `SwisstableDebugEnabled()` is also true for release builds with hardening + // enabled. To minimize their impact in those builds: + // - use `ABSL_PREDICT_FALSE()` to provide a compiler hint for code layout + // - use `ABSL_RAW_LOG()` with a format string to reduce code size and improve + // the chances that the hot paths will be inlined. + if (ABSL_PREDICT_FALSE(ctrl == nullptr)) { + ABSL_RAW_LOG(FATAL, "%s called on end() iterator.", operation); + } + if (ABSL_PREDICT_FALSE(ctrl == EmptyGroup())) { + ABSL_RAW_LOG(FATAL, "%s called on default-constructed iterator.", + operation); } if (SwisstableGenerationsEnabled()) { - if (generation != *generation_ptr) { - ABSL_INTERNAL_LOG(FATAL, - std::string(operation) + - " called on invalid iterator. The table could have " - "rehashed since this iterator was initialized."); + if (ABSL_PREDICT_FALSE(generation != *generation_ptr)) { + ABSL_RAW_LOG(FATAL, + "%s called on invalid iterator. The table could have " + "rehashed or moved since this iterator was initialized.", + operation); } - if (!IsFull(*ctrl)) { - ABSL_INTERNAL_LOG( + if (ABSL_PREDICT_FALSE(!IsFull(*ctrl))) { + ABSL_RAW_LOG( FATAL, - std::string(operation) + - " called on invalid iterator. The element was likely erased."); + "%s called on invalid iterator. The element was likely erased.", + operation); } } else { - if (!IsFull(*ctrl)) { - ABSL_INTERNAL_LOG( + if (ABSL_PREDICT_FALSE(!IsFull(*ctrl))) { + ABSL_RAW_LOG( FATAL, - std::string(operation) + - " called on invalid iterator. The element might have been erased " - "or the table might have rehashed. Consider running with " - "--config=asan to diagnose rehashing issues."); + "%s called on invalid iterator. The element might have been erased " + "or the table might have rehashed. Consider running with " + "--config=asan to diagnose rehashing issues.", + operation); } } } @@ -1180,13 +1292,13 @@ inline void AssertIsValidForComparison(const ctrl_t* ctrl, const bool ctrl_is_valid_for_comparison = ctrl == nullptr || ctrl == EmptyGroup() || IsFull(*ctrl); if (SwisstableGenerationsEnabled()) { - if (generation != *generation_ptr) { - ABSL_INTERNAL_LOG(FATAL, - "Invalid iterator comparison. The table could have " - "rehashed since this iterator was initialized."); + if (ABSL_PREDICT_FALSE(generation != *generation_ptr)) { + ABSL_RAW_LOG(FATAL, + "Invalid iterator comparison. The table could have rehashed " + "or moved since this iterator was initialized."); } - if (!ctrl_is_valid_for_comparison) { - ABSL_INTERNAL_LOG( + if (ABSL_PREDICT_FALSE(!ctrl_is_valid_for_comparison)) { + ABSL_RAW_LOG( FATAL, "Invalid iterator comparison. The element was likely erased."); } } else { @@ -1226,10 +1338,15 @@ inline void AssertSameContainer(const ctrl_t* ctrl_a, const ctrl_t* ctrl_b, const GenerationType* generation_ptr_a, const GenerationType* generation_ptr_b) { if (!SwisstableDebugEnabled()) return; + // `SwisstableDebugEnabled()` is also true for release builds with hardening + // enabled. To minimize their impact in those builds: + // - use `ABSL_PREDICT_FALSE()` to provide a compiler hint for code layout + // - use `ABSL_RAW_LOG()` with a format string to reduce code size and improve + // the chances that the hot paths will be inlined. const bool a_is_default = ctrl_a == EmptyGroup(); const bool b_is_default = ctrl_b == EmptyGroup(); - if (a_is_default != b_is_default) { - ABSL_INTERNAL_LOG( + if (ABSL_PREDICT_FALSE(a_is_default != b_is_default)) { + ABSL_RAW_LOG( FATAL, "Invalid iterator comparison. Comparing default-constructed iterator " "with non-default-constructed iterator."); @@ -1237,36 +1354,36 @@ inline void AssertSameContainer(const ctrl_t* ctrl_a, const ctrl_t* ctrl_b, if (a_is_default && b_is_default) return; if (SwisstableGenerationsEnabled()) { - if (generation_ptr_a == generation_ptr_b) return; + if (ABSL_PREDICT_TRUE(generation_ptr_a == generation_ptr_b)) return; const bool a_is_empty = IsEmptyGeneration(generation_ptr_a); const bool b_is_empty = IsEmptyGeneration(generation_ptr_b); if (a_is_empty != b_is_empty) { - ABSL_INTERNAL_LOG(FATAL, - "Invalid iterator comparison. Comparing iterator from " - "a non-empty hashtable with an iterator from an empty " - "hashtable."); + ABSL_RAW_LOG(FATAL, + "Invalid iterator comparison. Comparing iterator from a " + "non-empty hashtable with an iterator from an empty " + "hashtable."); } if (a_is_empty && b_is_empty) { - ABSL_INTERNAL_LOG(FATAL, - "Invalid iterator comparison. Comparing iterators from " - "different empty hashtables."); + ABSL_RAW_LOG(FATAL, + "Invalid iterator comparison. Comparing iterators from " + "different empty hashtables."); } const bool a_is_end = ctrl_a == nullptr; const bool b_is_end = ctrl_b == nullptr; if (a_is_end || b_is_end) { - ABSL_INTERNAL_LOG(FATAL, - "Invalid iterator comparison. Comparing iterator with " - "an end() iterator from a different hashtable."); + ABSL_RAW_LOG(FATAL, + "Invalid iterator comparison. Comparing iterator with an " + "end() iterator from a different hashtable."); } - ABSL_INTERNAL_LOG(FATAL, - "Invalid iterator comparison. Comparing non-end() " - "iterators from different hashtables."); + ABSL_RAW_LOG(FATAL, + "Invalid iterator comparison. Comparing non-end() iterators " + "from different hashtables."); } else { ABSL_HARDENING_ASSERT( AreItersFromSameContainer(ctrl_a, ctrl_b, slot_a, slot_b) && "Invalid iterator comparison. The iterators may be from different " - "containers or the container might have rehashed. Consider running " - "with --config=asan to diagnose rehashing issues."); + "containers or the container might have rehashed or moved. Consider " + "running with --config=asan to diagnose issues."); } } @@ -1289,6 +1406,12 @@ struct FindInfo { // `ShouldInsertBackwards()` for small tables. inline bool is_small(size_t capacity) { return capacity < Group::kWidth - 1; } +// Whether a table fits entirely into a probing group. +// Arbitrary order of elements in such tables is correct. +inline bool is_single_group(size_t capacity) { + return capacity <= Group::kWidth; +} + // Begins a probing operation on `common.control`, using `hash`. inline probe_seq<Group::kWidth> probe(const ctrl_t* ctrl, const size_t capacity, size_t hash) { @@ -1310,7 +1433,7 @@ inline FindInfo find_first_non_full(const CommonFields& common, size_t hash) { auto seq = probe(common, hash); const ctrl_t* ctrl = common.control(); while (true) { - Group g{ctrl + seq.offset()}; + GroupEmptyOrDeleted g{ctrl + seq.offset()}; auto mask = g.MaskEmptyOrDeleted(); if (mask) { #if !defined(NDEBUG) @@ -1351,7 +1474,6 @@ inline void ResetCtrl(CommonFields& common, size_t slot_size) { capacity + 1 + NumClonedBytes()); ctrl[capacity] = ctrl_t::kSentinel; SanitizerPoisonMemoryRegion(common.slot_array(), slot_size * capacity); - ResetGrowthLeft(common); } // Sets `ctrl[i]` to `h`. @@ -1386,38 +1508,263 @@ constexpr size_t BackingArrayAlignment(size_t align_of_slot) { return (std::max)(align_of_slot, alignof(size_t)); } -template <typename Alloc, size_t SizeOfSlot, size_t AlignOfSlot> -ABSL_ATTRIBUTE_NOINLINE void InitializeSlots(CommonFields& c, Alloc alloc) { - assert(c.capacity()); - // Folks with custom allocators often make unwarranted assumptions about the - // behavior of their classes vis-a-vis trivial destructability and what - // calls they will or won't make. Avoid sampling for people with custom - // allocators to get us out of this mess. This is not a hard guarantee but - // a workaround while we plan the exact guarantee we want to provide. - const size_t sample_size = - (std::is_same<Alloc, std::allocator<char>>::value && - c.slot_array() == nullptr) - ? SizeOfSlot - : 0; - - const size_t cap = c.capacity(); - const size_t alloc_size = AllocSize(cap, SizeOfSlot, AlignOfSlot); - // growth_left (which is a size_t) is stored with the backing array. - char* mem = static_cast<char*>( - Allocate<BackingArrayAlignment(AlignOfSlot)>(&alloc, alloc_size)); - const GenerationType old_generation = c.generation(); - c.set_generation_ptr( - reinterpret_cast<GenerationType*>(mem + GenerationOffset(cap))); - c.set_generation(NextGeneration(old_generation)); - c.set_control(reinterpret_cast<ctrl_t*>(mem + ControlOffset())); - c.set_slots(mem + SlotOffset(cap, AlignOfSlot)); - ResetCtrl(c, SizeOfSlot); - if (sample_size) { - c.infoz() = Sample(sample_size); - } - c.infoz().RecordStorageChanged(c.size(), cap); +// Returns the address of the ith slot in slots where each slot occupies +// slot_size. +inline void* SlotAddress(void* slot_array, size_t slot, size_t slot_size) { + return reinterpret_cast<void*>(reinterpret_cast<char*>(slot_array) + + (slot * slot_size)); } +// Helper class to perform resize of the hash set. +// +// It contains special optimizations for small group resizes. +// See GrowIntoSingleGroupShuffleControlBytes for details. +class HashSetResizeHelper { + public: + explicit HashSetResizeHelper(CommonFields& c) + : old_ctrl_(c.control()), + old_capacity_(c.capacity()), + had_infoz_(c.has_infoz()) {} + + // Optimized for small groups version of `find_first_non_full` applicable + // only right after calling `raw_hash_set::resize`. + // It has implicit assumption that `resize` will call + // `GrowSizeIntoSingleGroup*` in case `IsGrowingIntoSingleGroupApplicable`. + // Falls back to `find_first_non_full` in case of big groups, so it is + // safe to use after `rehash_and_grow_if_necessary`. + static FindInfo FindFirstNonFullAfterResize(const CommonFields& c, + size_t old_capacity, + size_t hash) { + if (!IsGrowingIntoSingleGroupApplicable(old_capacity, c.capacity())) { + return find_first_non_full(c, hash); + } + // Find a location for the new element non-deterministically. + // Note that any position is correct. + // It will located at `half_old_capacity` or one of the other + // empty slots with approximately 50% probability each. + size_t offset = probe(c, hash).offset(); + + // Note that we intentionally use unsigned int underflow. + if (offset - (old_capacity + 1) >= old_capacity) { + // Offset fall on kSentinel or into the mostly occupied first half. + offset = old_capacity / 2; + } + assert(IsEmpty(c.control()[offset])); + return FindInfo{offset, 0}; + } + + ctrl_t* old_ctrl() const { return old_ctrl_; } + size_t old_capacity() const { return old_capacity_; } + + // Allocates a backing array for the hashtable. + // Reads `capacity` and updates all other fields based on the result of + // the allocation. + // + // It also may do the folowing actions: + // 1. initialize control bytes + // 2. initialize slots + // 3. deallocate old slots. + // + // We are bundling a lot of functionality + // in one ABSL_ATTRIBUTE_NOINLINE function in order to minimize binary code + // duplication in raw_hash_set<>::resize. + // + // `c.capacity()` must be nonzero. + // POSTCONDITIONS: + // 1. CommonFields is initialized. + // + // if IsGrowingIntoSingleGroupApplicable && TransferUsesMemcpy + // Both control bytes and slots are fully initialized. + // old_slots are deallocated. + // infoz.RecordRehash is called. + // + // if IsGrowingIntoSingleGroupApplicable && !TransferUsesMemcpy + // Control bytes are fully initialized. + // infoz.RecordRehash is called. + // GrowSizeIntoSingleGroup must be called to finish slots initialization. + // + // if !IsGrowingIntoSingleGroupApplicable + // Control bytes are initialized to empty table via ResetCtrl. + // raw_hash_set<>::resize must insert elements regularly. + // infoz.RecordRehash is called if old_capacity == 0. + // + // Returns IsGrowingIntoSingleGroupApplicable result to avoid recomputation. + template <typename Alloc, size_t SizeOfSlot, bool TransferUsesMemcpy, + size_t AlignOfSlot> + ABSL_ATTRIBUTE_NOINLINE bool InitializeSlots(CommonFields& c, void* old_slots, + Alloc alloc) { + assert(c.capacity()); + // Folks with custom allocators often make unwarranted assumptions about the + // behavior of their classes vis-a-vis trivial destructability and what + // calls they will or won't make. Avoid sampling for people with custom + // allocators to get us out of this mess. This is not a hard guarantee but + // a workaround while we plan the exact guarantee we want to provide. + const size_t sample_size = + (std::is_same<Alloc, std::allocator<char>>::value && + c.slot_array() == nullptr) + ? SizeOfSlot + : 0; + HashtablezInfoHandle infoz = + sample_size > 0 ? Sample(sample_size) : c.infoz(); + + const bool has_infoz = infoz.IsSampled(); + const size_t cap = c.capacity(); + const size_t alloc_size = + AllocSize(cap, SizeOfSlot, AlignOfSlot, has_infoz); + char* mem = static_cast<char*>( + Allocate<BackingArrayAlignment(AlignOfSlot)>(&alloc, alloc_size)); + const GenerationType old_generation = c.generation(); + c.set_generation_ptr(reinterpret_cast<GenerationType*>( + mem + GenerationOffset(cap, has_infoz))); + c.set_generation(NextGeneration(old_generation)); + c.set_control(reinterpret_cast<ctrl_t*>(mem + ControlOffset(has_infoz))); + c.set_slots(mem + SlotOffset(cap, AlignOfSlot, has_infoz)); + ResetGrowthLeft(c); + + const bool grow_single_group = + IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity()); + if (old_capacity_ != 0 && grow_single_group) { + if (TransferUsesMemcpy) { + GrowSizeIntoSingleGroupTransferable(c, old_slots, SizeOfSlot); + DeallocateOld<AlignOfSlot>(alloc, SizeOfSlot, old_slots); + } else { + GrowIntoSingleGroupShuffleControlBytes(c.control(), c.capacity()); + } + } else { + ResetCtrl(c, SizeOfSlot); + } + + c.set_has_infoz(has_infoz); + if (has_infoz) { + infoz.RecordStorageChanged(c.size(), cap); + if (grow_single_group || old_capacity_ == 0) { + infoz.RecordRehash(0); + } + c.set_infoz(infoz); + } + return grow_single_group; + } + + // Relocates slots into new single group consistent with + // GrowIntoSingleGroupShuffleControlBytes. + // + // PRECONDITIONS: + // 1. GrowIntoSingleGroupShuffleControlBytes was already called. + template <class PolicyTraits, class Alloc> + void GrowSizeIntoSingleGroup(CommonFields& c, Alloc& alloc_ref, + typename PolicyTraits::slot_type* old_slots) { + assert(old_capacity_ < Group::kWidth / 2); + assert(IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity())); + using slot_type = typename PolicyTraits::slot_type; + assert(is_single_group(c.capacity())); + + auto* new_slots = reinterpret_cast<slot_type*>(c.slot_array()); + + size_t shuffle_bit = old_capacity_ / 2 + 1; + for (size_t i = 0; i < old_capacity_; ++i) { + if (IsFull(old_ctrl_[i])) { + size_t new_i = i ^ shuffle_bit; + SanitizerUnpoisonMemoryRegion(new_slots + new_i, sizeof(slot_type)); + PolicyTraits::transfer(&alloc_ref, new_slots + new_i, old_slots + i); + } + } + PoisonSingleGroupEmptySlots(c, sizeof(slot_type)); + } + + // Deallocates old backing array. + template <size_t AlignOfSlot, class CharAlloc> + void DeallocateOld(CharAlloc alloc_ref, size_t slot_size, void* old_slots) { + SanitizerUnpoisonMemoryRegion(old_slots, slot_size * old_capacity_); + Deallocate<BackingArrayAlignment(AlignOfSlot)>( + &alloc_ref, old_ctrl_ - ControlOffset(had_infoz_), + AllocSize(old_capacity_, slot_size, AlignOfSlot, had_infoz_)); + } + + private: + // Returns true if `GrowSizeIntoSingleGroup` can be used for resizing. + static bool IsGrowingIntoSingleGroupApplicable(size_t old_capacity, + size_t new_capacity) { + // NOTE that `old_capacity < new_capacity` in order to have + // `old_capacity < Group::kWidth / 2` to make faster copies of 8 bytes. + return is_single_group(new_capacity) && old_capacity < new_capacity; + } + + // Relocates control bytes and slots into new single group for + // transferable objects. + // Must be called only if IsGrowingIntoSingleGroupApplicable returned true. + void GrowSizeIntoSingleGroupTransferable(CommonFields& c, void* old_slots, + size_t slot_size); + + // Shuffle control bits deterministically to the next capacity. + // Returns offset for newly added element with given hash. + // + // PRECONDITIONs: + // 1. new_ctrl is allocated for new_capacity, + // but not initialized. + // 2. new_capacity is a single group. + // + // All elements are transferred into the first `old_capacity + 1` positions + // of the new_ctrl. Elements are rotated by `old_capacity_ / 2 + 1` positions + // in order to change an order and keep it non deterministic. + // Although rotation itself deterministic, position of the new added element + // will be based on `H1` and is not deterministic. + // + // Examples: + // S = kSentinel, E = kEmpty + // + // old_ctrl = SEEEEEEEE... + // new_ctrl = ESEEEEEEE... + // + // old_ctrl = 0SEEEEEEE... + // new_ctrl = E0ESE0EEE... + // + // old_ctrl = 012S012EEEEEEEEE... + // new_ctrl = 2E01EEES2E01EEE... + // + // old_ctrl = 0123456S0123456EEEEEEEEEEE... + // new_ctrl = 456E0123EEEEEES456E0123EEE... + void GrowIntoSingleGroupShuffleControlBytes(ctrl_t* new_ctrl, + size_t new_capacity) const; + + // Shuffle trivially transferable slots in the way consistent with + // GrowIntoSingleGroupShuffleControlBytes. + // + // PRECONDITIONs: + // 1. old_capacity must be non-zero. + // 2. new_ctrl is fully initialized using + // GrowIntoSingleGroupShuffleControlBytes. + // 3. new_slots is allocated and *not* poisoned. + // + // POSTCONDITIONS: + // 1. new_slots are transferred from old_slots_ consistent with + // GrowIntoSingleGroupShuffleControlBytes. + // 2. Empty new_slots are *not* poisoned. + void GrowIntoSingleGroupShuffleTransferableSlots(void* old_slots, + void* new_slots, + size_t slot_size) const; + + // Poison empty slots that were transferred using the deterministic algorithm + // described above. + // PRECONDITIONs: + // 1. new_ctrl is fully initialized using + // GrowIntoSingleGroupShuffleControlBytes. + // 2. new_slots is fully initialized consistent with + // GrowIntoSingleGroupShuffleControlBytes. + void PoisonSingleGroupEmptySlots(CommonFields& c, size_t slot_size) const { + // poison non full items + for (size_t i = 0; i < c.capacity(); ++i) { + if (!IsFull(c.control()[i])) { + SanitizerPoisonMemoryRegion(SlotAddress(c.slot_array(), i, slot_size), + slot_size); + } + } + } + + ctrl_t* old_ctrl_; + size_t old_capacity_; + bool had_infoz_; +}; + // PolicyFunctions bundles together some information for a particular // raw_hash_set<T, ...> instantiation. This information is passed to // type-erased functions that want to do small amounts of type-specific @@ -1442,7 +1789,7 @@ void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy, bool reuse); // Type-erased version of raw_hash_set::erase_meta_only. -void EraseMetaOnly(CommonFields& c, ctrl_t* it, size_t slot_size); +void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size); // Function to place in PolicyFunctions::dealloc for raw_hash_sets // that are using std::allocator. This allows us to share the same @@ -1456,6 +1803,7 @@ ABSL_ATTRIBUTE_NOINLINE void DeallocateStandard(CommonFields& common, policy.slot_size * common.capacity()); std::allocator<char> alloc; + common.infoz().Unregister(); Deallocate<BackingArrayAlignment(AlignOfSlot)>( &alloc, common.backing_array_start(), common.alloc_size(policy.slot_size, AlignOfSlot)); @@ -1534,6 +1882,11 @@ class raw_hash_set { using AllocTraits = absl::allocator_traits<allocator_type>; using SlotAlloc = typename absl::allocator_traits< allocator_type>::template rebind_alloc<slot_type>; + // People are often sloppy with the exact type of their allocator (sometimes + // it has an extra const or is missing the pair, but rebinds made it work + // anyway). + using CharAlloc = + typename absl::allocator_traits<Alloc>::template rebind_alloc<char>; using SlotAllocTraits = typename absl::allocator_traits< allocator_type>::template rebind_traits<slot_type>; @@ -1590,7 +1943,7 @@ class raw_hash_set { // PRECONDITION: not an end() iterator. reference operator*() const { AssertIsFull(ctrl_, generation(), generation_ptr(), "operator*()"); - return PolicyTraits::element(slot_); + return unchecked_deref(); } // PRECONDITION: not an end() iterator. @@ -1645,13 +1998,17 @@ class raw_hash_set { // If a sentinel is reached, we null `ctrl_` out instead. void skip_empty_or_deleted() { while (IsEmptyOrDeleted(*ctrl_)) { - uint32_t shift = Group{ctrl_}.CountLeadingEmptyOrDeleted(); + uint32_t shift = + GroupEmptyOrDeleted{ctrl_}.CountLeadingEmptyOrDeleted(); ctrl_ += shift; slot_ += shift; } if (ABSL_PREDICT_FALSE(*ctrl_ == ctrl_t::kSentinel)) ctrl_ = nullptr; } + ctrl_t* control() const { return ctrl_; } + slot_type* slot() const { return slot_; } + // We use EmptyGroup() for default-constructed iterators so that they can // be distinguished from end iterators, which have nullptr ctrl_. ctrl_t* ctrl_ = EmptyGroup(); @@ -1660,10 +2017,23 @@ class raw_hash_set { union { slot_type* slot_; }; + + // An equality check which skips ABSL Hardening iterator invalidation + // checks. + // Should be used when the lifetimes of the iterators are well-enough + // understood to prove that they cannot be invalid. + bool unchecked_equals(const iterator& b) { return ctrl_ == b.control(); } + + // Dereferences the iterator without ABSL Hardening iterator invalidation + // checks. + reference unchecked_deref() const { return PolicyTraits::element(slot_); } }; class const_iterator { friend class raw_hash_set; + template <class Container, typename Enabler> + friend struct absl::container_internal::hashtable_debug_internal:: + HashtableDebugAccess; public: using iterator_category = typename iterator::iterator_category; @@ -1697,8 +2067,14 @@ class raw_hash_set { const GenerationType* gen) : inner_(const_cast<ctrl_t*>(ctrl), const_cast<slot_type*>(slot), gen) { } + ctrl_t* control() const { return inner_.control(); } + slot_type* slot() const { return inner_.slot(); } iterator inner_; + + bool unchecked_equals(const const_iterator& b) { + return inner_.unchecked_equals(b.inner_); + } }; using node_type = node_handle<Policy, hash_policy_traits<Policy>, Alloc>; @@ -1717,8 +2093,7 @@ class raw_hash_set { const allocator_type& alloc = allocator_type()) : settings_(CommonFields{}, hash, eq, alloc) { if (bucket_count) { - common().set_capacity(NormalizeCapacity(bucket_count)); - initialize_slots(); + resize(NormalizeCapacity(bucket_count)); } } @@ -1843,28 +2218,35 @@ class raw_hash_set { : // Hash, equality and allocator are copied instead of moved because // `that` must be left valid. If Hash is std::function<Key>, moving it // would create a nullptr functor that cannot be called. - settings_(absl::exchange(that.common(), CommonFields{}), - that.hash_ref(), that.eq_ref(), that.alloc_ref()) {} + // TODO(b/296061262): move instead of copying hash/eq/alloc. + // Note: we avoid using exchange for better generated code. + settings_(std::move(that.common()), that.hash_ref(), that.eq_ref(), + that.alloc_ref()) { + that.common() = CommonFields{}; + maybe_increment_generation_or_rehash_on_move(); + } raw_hash_set(raw_hash_set&& that, const allocator_type& a) : settings_(CommonFields{}, that.hash_ref(), that.eq_ref(), a) { if (a == that.alloc_ref()) { std::swap(common(), that.common()); + maybe_increment_generation_or_rehash_on_move(); } else { - reserve(that.size()); - // Note: this will copy elements of dense_set and unordered_set instead of - // moving them. This can be fixed if it ever becomes an issue. - for (auto& elem : that) insert(std::move(elem)); + move_elements_allocs_unequal(std::move(that)); } } raw_hash_set& operator=(const raw_hash_set& that) { - raw_hash_set tmp(that, - AllocTraits::propagate_on_container_copy_assignment::value - ? that.alloc_ref() - : alloc_ref()); - swap(tmp); - return *this; + if (ABSL_PREDICT_FALSE(this == &that)) return *this; + constexpr bool propagate_alloc = + AllocTraits::propagate_on_container_copy_assignment::value; + // TODO(ezb): maybe avoid allocating a new backing array if this->capacity() + // is an exact match for that.size(). If this->capacity() is too big, then + // it would make iteration very slow to reuse the allocation. Maybe we can + // do the same heuristic as clear() and reuse if it's small enough. + raw_hash_set tmp(that, propagate_alloc ? that.alloc_ref() : alloc_ref()); + // NOLINTNEXTLINE: not returning *this for performance. + return assign_impl<propagate_alloc>(std::move(tmp)); } raw_hash_set& operator=(raw_hash_set&& that) noexcept( @@ -1879,19 +2261,7 @@ class raw_hash_set { typename AllocTraits::propagate_on_container_move_assignment()); } - ~raw_hash_set() { - const size_t cap = capacity(); - if (!cap) return; - destroy_slots(); - - // Unpoison before returning the memory to the allocator. - SanitizerUnpoisonMemoryRegion(slot_array(), sizeof(slot_type) * cap); - Deallocate<BackingArrayAlignment(alignof(slot_type))>( - &alloc_ref(), common().backing_array_start(), - AllocSize(cap, sizeof(slot_type), alignof(slot_type))); - - infoz().Unregister(); - } + ~raw_hash_set() { destructor_impl(); } iterator begin() ABSL_ATTRIBUTE_LIFETIME_BOUND { auto it = iterator_at(0); @@ -1937,17 +2307,6 @@ class raw_hash_set { common().set_reservation_size(0); } - inline void destroy_slots() { - const size_t cap = capacity(); - const ctrl_t* ctrl = control(); - slot_type* slot = slot_array(); - for (size_t i = 0; i != cap; ++i) { - if (IsFull(ctrl[i])) { - PolicyTraits::destroy(&alloc_ref(), slot + i); - } - } - } - // This overload kicks in when the argument is an rvalue of insertable and // decomposable type other than init_type. // @@ -2075,7 +2434,7 @@ class raw_hash_set { alignas(slot_type) unsigned char raw[sizeof(slot_type)]; slot_type* slot = reinterpret_cast<slot_type*>(&raw); - PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...); + construct(slot, std::forward<Args>(args)...); const auto& elem = PolicyTraits::element(slot); return PolicyTraits::apply(InsertSlot<true>{*this, std::move(*slot)}, elem); } @@ -2179,8 +2538,8 @@ class raw_hash_set { // This overload is necessary because otherwise erase<K>(const K&) would be // a better match if non-const iterator is passed as an argument. void erase(iterator it) { - AssertIsFull(it.ctrl_, it.generation(), it.generation_ptr(), "erase()"); - PolicyTraits::destroy(&alloc_ref(), it.slot_); + AssertIsFull(it.control(), it.generation(), it.generation_ptr(), "erase()"); + destroy(it.slot()); erase_meta_only(it); } @@ -2211,8 +2570,8 @@ class raw_hash_set { assert(this != &src); for (auto it = src.begin(), e = src.end(); it != e;) { auto next = std::next(it); - if (PolicyTraits::apply(InsertSlot<false>{*this, std::move(*it.slot_)}, - PolicyTraits::element(it.slot_)) + if (PolicyTraits::apply(InsertSlot<false>{*this, std::move(*it.slot())}, + PolicyTraits::element(it.slot())) .second) { src.erase_meta_only(it); } @@ -2226,10 +2585,9 @@ class raw_hash_set { } node_type extract(const_iterator position) { - AssertIsFull(position.inner_.ctrl_, position.inner_.generation(), + AssertIsFull(position.control(), position.inner_.generation(), position.inner_.generation_ptr(), "extract()"); - auto node = - CommonAccess::Transfer<node_type>(alloc_ref(), position.inner_.slot_); + auto node = CommonAccess::Transfer<node_type>(alloc_ref(), position.slot()); erase_meta_only(position); return node; } @@ -2364,7 +2722,11 @@ class raw_hash_set { template <class K = key_type> bool contains(const key_arg<K>& key) const { - return find(key) != end(); + // Here neither the iterator returned by `find()` nor `end()` can be invalid + // outside of potential thread-safety issues. + // `find()`'s return value is constructed, used, and then destructed + // all in this context. + return !find(key).unchecked_equals(end()); } template <class K = key_type> @@ -2400,8 +2762,10 @@ class raw_hash_set { const raw_hash_set* outer = &a; const raw_hash_set* inner = &b; if (outer->capacity() > inner->capacity()) std::swap(outer, inner); - for (const value_type& elem : *outer) - if (!inner->has_element(elem)) return false; + for (const value_type& elem : *outer) { + auto it = PolicyTraits::apply(FindElement{*inner}, elem); + if (it == inner->end() || !(*it == elem)) return false; + } return true; } @@ -2471,10 +2835,9 @@ class raw_hash_set { std::pair<iterator, bool> operator()(const K& key, Args&&...) && { auto res = s.find_or_prepare_insert(key); if (res.second) { - PolicyTraits::transfer(&s.alloc_ref(), s.slot_array() + res.first, - &slot); + s.transfer(s.slot_array() + res.first, &slot); } else if (do_destroy) { - PolicyTraits::destroy(&s.alloc_ref(), &slot); + s.destroy(&slot); } return {s.iterator_at(res.first), res.second}; } @@ -2483,58 +2846,111 @@ class raw_hash_set { slot_type&& slot; }; + // TODO(b/303305702): re-enable reentrant validation. + template <typename... Args> + inline void construct(slot_type* slot, Args&&... args) { + PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...); + } + inline void destroy(slot_type* slot) { + PolicyTraits::destroy(&alloc_ref(), slot); + } + inline void transfer(slot_type* to, slot_type* from) { + PolicyTraits::transfer(&alloc_ref(), to, from); + } + + inline void destroy_slots() { + const size_t cap = capacity(); + const ctrl_t* ctrl = control(); + slot_type* slot = slot_array(); + for (size_t i = 0; i != cap; ++i) { + if (IsFull(ctrl[i])) { + destroy(slot + i); + } + } + } + + inline void dealloc() { + assert(capacity() != 0); + // Unpoison before returning the memory to the allocator. + SanitizerUnpoisonMemoryRegion(slot_array(), sizeof(slot_type) * capacity()); + infoz().Unregister(); + Deallocate<BackingArrayAlignment(alignof(slot_type))>( + &alloc_ref(), common().backing_array_start(), + common().alloc_size(sizeof(slot_type), alignof(slot_type))); + } + + inline void destructor_impl() { + if (capacity() == 0) return; + destroy_slots(); + dealloc(); + } + // Erases, but does not destroy, the value pointed to by `it`. // // This merely updates the pertinent control byte. This can be used in // conjunction with Policy::transfer to move the object to another place. void erase_meta_only(const_iterator it) { - EraseMetaOnly(common(), it.inner_.ctrl_, sizeof(slot_type)); + EraseMetaOnly(common(), static_cast<size_t>(it.control() - control()), + sizeof(slot_type)); } - // Allocates a backing array for `self` and initializes its control bytes. - // This reads `capacity` and updates all other fields based on the result of - // the allocation. + // Resizes table to the new capacity and move all elements to the new + // positions accordingly. // - // This does not free the currently held array; `capacity` must be nonzero. - inline void initialize_slots() { - // People are often sloppy with the exact type of their allocator (sometimes - // it has an extra const or is missing the pair, but rebinds made it work - // anyway). - using CharAlloc = - typename absl::allocator_traits<Alloc>::template rebind_alloc<char>; - InitializeSlots<CharAlloc, sizeof(slot_type), alignof(slot_type)>( - common(), CharAlloc(alloc_ref())); - } - + // Note that for better performance instead of + // find_first_non_full(common(), hash), + // HashSetResizeHelper::FindFirstNonFullAfterResize( + // common(), old_capacity, hash) + // can be called right after `resize`. ABSL_ATTRIBUTE_NOINLINE void resize(size_t new_capacity) { assert(IsValidCapacity(new_capacity)); - auto* old_ctrl = control(); + HashSetResizeHelper resize_helper(common()); auto* old_slots = slot_array(); - const size_t old_capacity = common().capacity(); common().set_capacity(new_capacity); - initialize_slots(); - - auto* new_slots = slot_array(); - size_t total_probe_length = 0; - for (size_t i = 0; i != old_capacity; ++i) { - if (IsFull(old_ctrl[i])) { - size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, - PolicyTraits::element(old_slots + i)); - auto target = find_first_non_full(common(), hash); - size_t new_i = target.offset; - total_probe_length += target.probe_length; - SetCtrl(common(), new_i, H2(hash), sizeof(slot_type)); - PolicyTraits::transfer(&alloc_ref(), new_slots + new_i, old_slots + i); - } + // Note that `InitializeSlots` does different number initialization steps + // depending on the values of `transfer_uses_memcpy` and capacities. + // Refer to the comment in `InitializeSlots` for more details. + const bool grow_single_group = + resize_helper.InitializeSlots<CharAlloc, sizeof(slot_type), + PolicyTraits::transfer_uses_memcpy(), + alignof(slot_type)>( + common(), const_cast<std::remove_const_t<slot_type>*>(old_slots), + CharAlloc(alloc_ref())); + + if (resize_helper.old_capacity() == 0) { + // InitializeSlots did all the work including infoz().RecordRehash(). + return; } - if (old_capacity) { - SanitizerUnpoisonMemoryRegion(old_slots, - sizeof(slot_type) * old_capacity); - Deallocate<BackingArrayAlignment(alignof(slot_type))>( - &alloc_ref(), old_ctrl - ControlOffset(), - AllocSize(old_capacity, sizeof(slot_type), alignof(slot_type))); + + if (grow_single_group) { + if (PolicyTraits::transfer_uses_memcpy()) { + // InitializeSlots did all the work. + return; + } + // We want GrowSizeIntoSingleGroup to be called here in order to make + // InitializeSlots not depend on PolicyTraits. + resize_helper.GrowSizeIntoSingleGroup<PolicyTraits>(common(), alloc_ref(), + old_slots); + } else { + // InitializeSlots prepares control bytes to correspond to empty table. + auto* new_slots = slot_array(); + size_t total_probe_length = 0; + for (size_t i = 0; i != resize_helper.old_capacity(); ++i) { + if (IsFull(resize_helper.old_ctrl()[i])) { + size_t hash = PolicyTraits::apply( + HashElement{hash_ref()}, PolicyTraits::element(old_slots + i)); + auto target = find_first_non_full(common(), hash); + size_t new_i = target.offset; + total_probe_length += target.probe_length; + SetCtrl(common(), new_i, H2(hash), sizeof(slot_type)); + transfer(new_slots + new_i, old_slots + i); + } + } + infoz().RecordRehash(total_probe_length); } - infoz().RecordRehash(total_probe_length); + resize_helper.DeallocateOld<alignof(slot_type)>( + CharAlloc(alloc_ref()), sizeof(slot_type), + const_cast<std::remove_const_t<slot_type>*>(old_slots)); } // Prunes control bytes to remove as many tombstones as possible. @@ -2604,36 +3020,64 @@ class raw_hash_set { } } - bool has_element(const value_type& elem) const { - size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, elem); - auto seq = probe(common(), hash); - const ctrl_t* ctrl = control(); - while (true) { - Group g{ctrl + seq.offset()}; - for (uint32_t i : g.Match(H2(hash))) { - if (ABSL_PREDICT_TRUE( - PolicyTraits::element(slot_array() + seq.offset(i)) == elem)) - return true; - } - if (ABSL_PREDICT_TRUE(g.MaskEmpty())) return false; - seq.next(); - assert(seq.index() <= capacity() && "full table!"); + void maybe_increment_generation_or_rehash_on_move() { + common().maybe_increment_generation_on_move(); + if (!empty() && common().should_rehash_for_bug_detection_on_move()) { + resize(capacity()); } - return false; } - // TODO(alkis): Optimize this assuming *this and that don't overlap. - raw_hash_set& move_assign(raw_hash_set&& that, std::true_type) { - raw_hash_set tmp(std::move(that)); - swap(tmp); + template<bool propagate_alloc> + raw_hash_set& assign_impl(raw_hash_set&& that) { + // We don't bother checking for this/that aliasing. We just need to avoid + // breaking the invariants in that case. + destructor_impl(); + common() = std::move(that.common()); + // TODO(b/296061262): move instead of copying hash/eq/alloc. + hash_ref() = that.hash_ref(); + eq_ref() = that.eq_ref(); + CopyAlloc(alloc_ref(), that.alloc_ref(), + std::integral_constant<bool, propagate_alloc>()); + that.common() = CommonFields{}; + maybe_increment_generation_or_rehash_on_move(); return *this; } - raw_hash_set& move_assign(raw_hash_set&& that, std::false_type) { - raw_hash_set tmp(std::move(that), alloc_ref()); - swap(tmp); + + raw_hash_set& move_elements_allocs_unequal(raw_hash_set&& that) { + const size_t size = that.size(); + if (size == 0) return *this; + reserve(size); + for (iterator it = that.begin(); it != that.end(); ++it) { + insert(std::move(PolicyTraits::element(it.slot()))); + that.destroy(it.slot()); + } + that.dealloc(); + that.common() = CommonFields{}; + maybe_increment_generation_or_rehash_on_move(); return *this; } + raw_hash_set& move_assign(raw_hash_set&& that, + std::true_type /*propagate_alloc*/) { + return assign_impl<true>(std::move(that)); + } + raw_hash_set& move_assign(raw_hash_set&& that, + std::false_type /*propagate_alloc*/) { + if (alloc_ref() == that.alloc_ref()) { + return assign_impl<false>(std::move(that)); + } + // Aliasing can't happen here because allocs would compare equal above. + assert(this != &that); + destructor_impl(); + // We can't take over that's memory so we need to move each element. + // While moving elements, this should have that's hash/eq so copy hash/eq + // before moving elements. + // TODO(b/296061262): move instead of copying hash/eq. + hash_ref() = that.hash_ref(); + eq_ref() = that.eq_ref(); + return move_elements_allocs_unequal(std::move(that)); + } + protected: // Attempts to find `key` in the table; if it isn't found, returns a slot that // the value can be inserted into, with the control byte already set to @@ -2675,10 +3119,19 @@ class raw_hash_set { if (!rehash_for_bug_detection && ABSL_PREDICT_FALSE(growth_left() == 0 && !IsDeleted(control()[target.offset]))) { + size_t old_capacity = capacity(); rehash_and_grow_if_necessary(); - target = find_first_non_full(common(), hash); + // NOTE: It is safe to use `FindFirstNonFullAfterResize`. + // `FindFirstNonFullAfterResize` must be called right after resize. + // `rehash_and_grow_if_necessary` may *not* call `resize` + // and perform `drop_deletes_without_resize` instead. But this + // could happen only on big tables. + // For big tables `FindFirstNonFullAfterResize` will always + // fallback to normal `find_first_non_full`, so it is safe to use it. + target = HashSetResizeHelper::FindFirstNonFullAfterResize( + common(), old_capacity, hash); } - common().set_size(common().size() + 1); + common().increment_size(); set_growth_left(growth_left() - IsEmpty(control()[target.offset])); SetCtrl(common(), target.offset, H2(hash), sizeof(slot_type)); common().maybe_increment_generation_on_insert(); @@ -2696,8 +3149,7 @@ class raw_hash_set { // POSTCONDITION: *m.iterator_at(i) == value_type(forward<Args>(args)...). template <class... Args> void emplace_at(size_t i, Args&&... args) { - PolicyTraits::construct(&alloc_ref(), slot_array() + i, - std::forward<Args>(args)...); + construct(slot_array() + i, std::forward<Args>(args)...); assert(PolicyTraits::apply(FindElement{*this}, *iterator_at(i)) == iterator_at(i) && @@ -2711,6 +3163,8 @@ class raw_hash_set { return {control() + i, slot_array() + i, common().generation_ptr()}; } + reference unchecked_deref(iterator it) { return it.unchecked_deref(); } + private: friend struct RawHashSetTestOnlyAccess; @@ -2743,7 +3197,7 @@ class raw_hash_set { slot_type* slot_array() const { return static_cast<slot_type*>(common().slot_array()); } - HashtablezInfoHandle& infoz() { return common().infoz(); } + HashtablezInfoHandle infoz() { return common().infoz(); } hasher& hash_ref() { return settings_.template get<1>(); } const hasher& hash_ref() const { return settings_.template get<1>(); } @@ -2763,8 +3217,7 @@ class raw_hash_set { } static void transfer_slot_fn(void* set, void* dst, void* src) { auto* h = static_cast<raw_hash_set*>(set); - PolicyTraits::transfer(&h->alloc_ref(), static_cast<slot_type*>(dst), - static_cast<slot_type*>(src)); + h->transfer(static_cast<slot_type*>(dst), static_cast<slot_type*>(src)); } // Note: dealloc_fn will only be used if we have a non-standard allocator. static void dealloc_fn(CommonFields& common, const PolicyFunctions&) { @@ -2774,6 +3227,7 @@ class raw_hash_set { SanitizerUnpoisonMemoryRegion(common.slot_array(), sizeof(slot_type) * common.capacity()); + common.infoz().Unregister(); Deallocate<BackingArrayAlignment(alignof(slot_type))>( &set->alloc_ref(), common.backing_array_start(), common.alloc_size(sizeof(slot_type), alignof(slot_type))); @@ -2847,33 +3301,18 @@ struct HashtableDebugAccess<Set, absl::void_t<typename Set::raw_hash_set>> { static size_t AllocatedByteSize(const Set& c) { size_t capacity = c.capacity(); if (capacity == 0) return 0; - size_t m = AllocSize(capacity, sizeof(Slot), alignof(Slot)); + size_t m = c.common().alloc_size(sizeof(Slot), alignof(Slot)); size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr)); if (per_slot != ~size_t{}) { m += per_slot * c.size(); } else { - const ctrl_t* ctrl = c.control(); - for (size_t i = 0; i != capacity; ++i) { - if (container_internal::IsFull(ctrl[i])) { - m += Traits::space_used(c.slot_array() + i); - } + for (auto it = c.begin(); it != c.end(); ++it) { + m += Traits::space_used(it.slot()); } } return m; } - - static size_t LowerBoundAllocatedByteSize(size_t size) { - size_t capacity = GrowthToLowerboundCapacity(size); - if (capacity == 0) return 0; - size_t m = - AllocSize(NormalizeCapacity(capacity), sizeof(Slot), alignof(Slot)); - size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr)); - if (per_slot != ~size_t{}) { - m += per_slot * size; - } - return m; - } }; } // namespace hashtable_debug_internal diff --git a/contrib/restricted/abseil-cpp/absl/crc/internal/cpu_detect.cc b/contrib/restricted/abseil-cpp/absl/crc/internal/cpu_detect.cc index 838380854f..d7eedd1ca4 100644 --- a/contrib/restricted/abseil-cpp/absl/crc/internal/cpu_detect.cc +++ b/contrib/restricted/abseil-cpp/absl/crc/internal/cpu_detect.cc @@ -189,8 +189,14 @@ CpuType GetAmdCpuType() { break; case 0x19: switch (model_num) { + case 0x0: // Stepping Ax case 0x1: // Stepping B0 return CpuType::kAmdMilan; + case 0x10: // Stepping A0 + case 0x11: // Stepping B0 + return CpuType::kAmdGenoa; + case 0x44: // Stepping A0 + return CpuType::kAmdRyzenV3000; default: return CpuType::kUnknown; } @@ -237,8 +243,26 @@ CpuType GetCpuType() { ABSL_INTERNAL_AARCH64_ID_REG_READ(MIDR_EL1, midr); uint32_t implementer = (midr >> 24) & 0xff; uint32_t part_number = (midr >> 4) & 0xfff; - if (implementer == 0x41 && part_number == 0xd0c) { - return CpuType::kArmNeoverseN1; + switch (implementer) { + case 0x41: + switch (part_number) { + case 0xd0c: return CpuType::kArmNeoverseN1; + case 0xd40: return CpuType::kArmNeoverseV1; + case 0xd49: return CpuType::kArmNeoverseN2; + case 0xd4f: return CpuType::kArmNeoverseV2; + default: + return CpuType::kUnknown; + } + break; + case 0xc0: + switch (part_number) { + case 0xac3: return CpuType::kAmpereSiryn; + default: + return CpuType::kUnknown; + } + break; + default: + return CpuType::kUnknown; } } return CpuType::kUnknown; diff --git a/contrib/restricted/abseil-cpp/absl/crc/internal/cpu_detect.h b/contrib/restricted/abseil-cpp/absl/crc/internal/cpu_detect.h index 6054f6960d..01e19590ca 100644 --- a/contrib/restricted/abseil-cpp/absl/crc/internal/cpu_detect.h +++ b/contrib/restricted/abseil-cpp/absl/crc/internal/cpu_detect.h @@ -29,6 +29,8 @@ enum class CpuType { kAmdRome, kAmdNaples, kAmdMilan, + kAmdGenoa, + kAmdRyzenV3000, kIntelCascadelakeXeon, kIntelSkylakeXeon, kIntelBroadwell, @@ -37,6 +39,10 @@ enum class CpuType { kIntelSandybridge, kIntelWestmere, kArmNeoverseN1, + kArmNeoverseV1, + kAmpereSiryn, + kArmNeoverseN2, + kArmNeoverseV2 }; // Returns the type of host CPU this code is running on. Returns kUnknown if diff --git a/contrib/restricted/abseil-cpp/absl/crc/internal/crc32_x86_arm_combined_simd.h b/contrib/restricted/abseil-cpp/absl/crc/internal/crc32_x86_arm_combined_simd.h index 39e53dd08e..59995ae3e2 100644 --- a/contrib/restricted/abseil-cpp/absl/crc/internal/crc32_x86_arm_combined_simd.h +++ b/contrib/restricted/abseil-cpp/absl/crc/internal/crc32_x86_arm_combined_simd.h @@ -59,6 +59,8 @@ namespace crc_internal { #if defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) using V128 = uint64x2_t; #else +// Note: Do not use __m128i_u, it is not portable. +// Use V128_LoadU() perform an unaligned load from __m128i*. using V128 = __m128i; #endif @@ -78,6 +80,9 @@ V128 V128_Load(const V128* src); // Load 128 bits of integer data. |src| does not need to be aligned. V128 V128_LoadU(const V128* src); +// Store 128 bits of integer data. |src| must be 16-byte aligned. +void V128_Store(V128* dst, V128 data); + // Polynomially multiplies the high 64 bits of |l| and |r|. V128 V128_PMulHi(const V128 l, const V128 r); @@ -109,6 +114,10 @@ V128 V128_ShiftRight(const V128 l); template <int imm> int V128_Extract32(const V128 l); +// Extracts a 64-bit integer from |l|, selected with |imm|. +template <int imm> +uint64_t V128_Extract64(const V128 l); + // Extracts the low 64 bits from V128. int64_t V128_Low64(const V128 l); @@ -139,6 +148,8 @@ inline V128 V128_Load(const V128* src) { return _mm_load_si128(src); } inline V128 V128_LoadU(const V128* src) { return _mm_loadu_si128(src); } +inline void V128_Store(V128* dst, V128 data) { _mm_store_si128(dst, data); } + inline V128 V128_PMulHi(const V128 l, const V128 r) { return _mm_clmulepi64_si128(l, r, 0x11); } @@ -173,6 +184,11 @@ inline int V128_Extract32(const V128 l) { return _mm_extract_epi32(l, imm); } +template <int imm> +inline uint64_t V128_Extract64(const V128 l) { + return static_cast<uint64_t>(_mm_extract_epi64(l, imm)); +} + inline int64_t V128_Low64(const V128 l) { return _mm_cvtsi128_si64(l); } inline V128 V128_ShiftLeft64(const V128 l, const V128 r) { @@ -203,10 +219,14 @@ inline V128 V128_LoadU(const V128* src) { return vld1q_u64(reinterpret_cast<const uint64_t*>(src)); } +inline void V128_Store(V128* dst, V128 data) { + vst1q_u64(reinterpret_cast<uint64_t*>(dst), data); +} + // Using inline assembly as clang does not generate the pmull2 instruction and // performance drops by 15-20%. -// TODO(b/193678732): Investigate why the compiler decides not to generate -// such instructions and why it becomes so much worse. +// TODO(b/193678732): Investigate why there is a slight performance hit when +// using intrinsics instead of inline assembly. inline V128 V128_PMulHi(const V128 l, const V128 r) { uint64x2_t res; __asm__ __volatile__("pmull2 %0.1q, %1.2d, %2.2d \n\t" @@ -215,10 +235,14 @@ inline V128 V128_PMulHi(const V128 l, const V128 r) { return res; } +// TODO(b/193678732): Investigate why the compiler decides to move the constant +// loop multiplicands from GPR to Neon registers every loop iteration. inline V128 V128_PMulLow(const V128 l, const V128 r) { - return reinterpret_cast<V128>(vmull_p64( - reinterpret_cast<poly64_t>(vget_low_p64(vreinterpretq_p64_u64(l))), - reinterpret_cast<poly64_t>(vget_low_p64(vreinterpretq_p64_u64(r))))); + uint64x2_t res; + __asm__ __volatile__("pmull %0.1q, %1.1d, %2.1d \n\t" + : "=w"(res) + : "w"(l), "w"(r)); + return res; } inline V128 V128_PMul01(const V128 l, const V128 r) { @@ -252,6 +276,11 @@ inline int V128_Extract32(const V128 l) { return vgetq_lane_s32(vreinterpretq_s32_u64(l), imm); } +template <int imm> +inline uint64_t V128_Extract64(const V128 l) { + return vgetq_lane_u64(l, imm); +} + inline int64_t V128_Low64(const V128 l) { return vgetq_lane_s64(vreinterpretq_s64_u64(l), 0); } diff --git a/contrib/restricted/abseil-cpp/absl/crc/internal/crc_memcpy.h b/contrib/restricted/abseil-cpp/absl/crc/internal/crc_memcpy.h index 4909d43366..a0fed65afa 100644 --- a/contrib/restricted/abseil-cpp/absl/crc/internal/crc_memcpy.h +++ b/contrib/restricted/abseil-cpp/absl/crc/internal/crc_memcpy.h @@ -20,12 +20,15 @@ #include "absl/base/config.h" #include "absl/crc/crc32c.h" +#include "absl/crc/internal/crc32_x86_arm_combined_simd.h" // Defined if the class AcceleratedCrcMemcpyEngine exists. -#if defined(__x86_64__) && defined(__SSE4_2__) -#define ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE 1 -#elif defined(_MSC_VER) && defined(__AVX__) +// TODO(b/299127771): Consider relaxing the pclmul requirement once the other +// intrinsics are conditionally compiled without it. +#if defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) #define ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE 1 +#elif defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) +#define ABSL_INTERNAL_HAVE_ARM_ACCELERATED_CRC_MEMCPY_ENGINE 1 #endif namespace absl { diff --git a/contrib/restricted/abseil-cpp/absl/crc/internal/crc_memcpy_fallback.cc b/contrib/restricted/abseil-cpp/absl/crc/internal/crc_memcpy_fallback.cc index 15b4b05594..07795504e3 100644 --- a/contrib/restricted/abseil-cpp/absl/crc/internal/crc_memcpy_fallback.cc +++ b/contrib/restricted/abseil-cpp/absl/crc/internal/crc_memcpy_fallback.cc @@ -54,7 +54,8 @@ absl::crc32c_t FallbackCrcMemcpyEngine::Compute(void* __restrict dst, } // Compile the following only if we don't have -#ifndef ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE +#if !defined(ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE) && \ + !defined(ABSL_INTERNAL_HAVE_ARM_ACCELERATED_CRC_MEMCPY_ENGINE) CrcMemcpy::ArchSpecificEngines CrcMemcpy::GetArchSpecificEngines() { CrcMemcpy::ArchSpecificEngines engines; @@ -68,7 +69,8 @@ std::unique_ptr<CrcMemcpyEngine> CrcMemcpy::GetTestEngine(int /*vector*/, return std::make_unique<FallbackCrcMemcpyEngine>(); } -#endif // ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE +#endif // !ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE && + // !ABSL_INTERNAL_HAVE_ARM_ACCELERATED_CRC_MEMCPY_ENGINE } // namespace crc_internal ABSL_NAMESPACE_END diff --git a/contrib/restricted/abseil-cpp/absl/crc/internal/crc_memcpy_x86_64.cc b/contrib/restricted/abseil-cpp/absl/crc/internal/crc_memcpy_x86_arm_combined.cc index d42b08dc9f..968e9ae359 100644 --- a/contrib/restricted/abseil-cpp/absl/crc/internal/crc_memcpy_x86_64.cc +++ b/contrib/restricted/abseil-cpp/absl/crc/internal/crc_memcpy_x86_arm_combined.cc @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Simultaneous memcopy and CRC-32C for x86-64. Uses integer registers because -// XMM registers do not support the CRC instruction (yet). While copying, -// compute the running CRC of the data being copied. +// Simultaneous memcopy and CRC-32C for x86-64 and ARM 64. Uses integer +// registers because XMM registers do not support the CRC instruction (yet). +// While copying, compute the running CRC of the data being copied. // // It is assumed that any CPU running this code has SSE4.2 instructions // available (for CRC32C). This file will do nothing if that is not true. @@ -49,17 +49,20 @@ #include <array> #include <cstddef> #include <cstdint> -#include <type_traits> +#include <cstring> +#include <memory> -#include "absl/base/dynamic_annotations.h" +#include "absl/base/config.h" #include "absl/base/optimization.h" #include "absl/base/prefetch.h" #include "absl/crc/crc32c.h" #include "absl/crc/internal/cpu_detect.h" +#include "absl/crc/internal/crc32_x86_arm_combined_simd.h" #include "absl/crc/internal/crc_memcpy.h" #include "absl/strings/string_view.h" -#ifdef ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE +#if defined(ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE) || \ + defined(ABSL_INTERNAL_HAVE_ARM_ACCELERATED_CRC_MEMCPY_ENGINE) namespace absl { ABSL_NAMESPACE_BEGIN @@ -74,7 +77,7 @@ inline crc32c_t ShortCrcCopy(char* dst, const char* src, std::size_t length, uint32_t crc_uint32 = static_cast<uint32_t>(crc); for (std::size_t i = 0; i < length; i++) { uint8_t data = *reinterpret_cast<const uint8_t*>(src); - crc_uint32 = _mm_crc32_u8(crc_uint32, data); + crc_uint32 = CRC32_u8(crc_uint32, data); *reinterpret_cast<uint8_t*>(dst) = data; ++src; ++dst; @@ -82,36 +85,35 @@ inline crc32c_t ShortCrcCopy(char* dst, const char* src, std::size_t length, return crc32c_t{crc_uint32}; } -constexpr size_t kIntLoadsPerVec = sizeof(__m128i) / sizeof(uint64_t); +constexpr size_t kIntLoadsPerVec = sizeof(V128) / sizeof(uint64_t); // Common function for copying the tails of multiple large regions. template <size_t vec_regions, size_t int_regions> inline void LargeTailCopy(crc32c_t* crcs, char** dst, const char** src, size_t region_size, size_t copy_rounds) { - std::array<__m128i, vec_regions> data; + std::array<V128, vec_regions> data; std::array<uint64_t, kIntLoadsPerVec * int_regions> int_data; while (copy_rounds > 0) { for (size_t i = 0; i < vec_regions; i++) { size_t region = i; - auto* vsrc = - reinterpret_cast<const __m128i*>(*src + region_size * region); - auto* vdst = reinterpret_cast<__m128i*>(*dst + region_size * region); + auto* vsrc = reinterpret_cast<const V128*>(*src + region_size * region); + auto* vdst = reinterpret_cast<V128*>(*dst + region_size * region); // Load the blocks, unaligned - data[i] = _mm_loadu_si128(vsrc); + data[i] = V128_LoadU(vsrc); // Store the blocks, aligned - _mm_store_si128(vdst, data[i]); + V128_Store(vdst, data[i]); // Compute the running CRC crcs[region] = crc32c_t{static_cast<uint32_t>( - _mm_crc32_u64(static_cast<uint32_t>(crcs[region]), - static_cast<uint64_t>(_mm_extract_epi64(data[i], 0))))}; + CRC32_u64(static_cast<uint32_t>(crcs[region]), + static_cast<uint64_t>(V128_Extract64<0>(data[i]))))}; crcs[region] = crc32c_t{static_cast<uint32_t>( - _mm_crc32_u64(static_cast<uint32_t>(crcs[region]), - static_cast<uint64_t>(_mm_extract_epi64(data[i], 1))))}; + CRC32_u64(static_cast<uint32_t>(crcs[region]), + static_cast<uint64_t>(V128_Extract64<1>(data[i]))))}; } for (size_t i = 0; i < int_regions; i++) { @@ -125,7 +127,7 @@ inline void LargeTailCopy(crc32c_t* crcs, char** dst, const char** src, size_t data_index = i * kIntLoadsPerVec + j; int_data[data_index] = *(usrc + j); - crcs[region] = crc32c_t{static_cast<uint32_t>(_mm_crc32_u64( + crcs[region] = crc32c_t{static_cast<uint32_t>(CRC32_u64( static_cast<uint32_t>(crcs[region]), int_data[data_index]))}; *(udst + j) = int_data[data_index]; @@ -133,8 +135,8 @@ inline void LargeTailCopy(crc32c_t* crcs, char** dst, const char** src, } // Increment pointers - *src += sizeof(__m128i); - *dst += sizeof(__m128i); + *src += sizeof(V128); + *dst += sizeof(V128); --copy_rounds; } } @@ -158,8 +160,9 @@ crc32c_t AcceleratedCrcMemcpyEngine<vec_regions, int_regions>::Compute( void* __restrict dst, const void* __restrict src, std::size_t length, crc32c_t initial_crc) const { constexpr std::size_t kRegions = vec_regions + int_regions; + static_assert(kRegions > 0, "Must specify at least one region."); constexpr uint32_t kCrcDataXor = uint32_t{0xffffffff}; - constexpr std::size_t kBlockSize = sizeof(__m128i); + constexpr std::size_t kBlockSize = sizeof(V128); constexpr std::size_t kCopyRoundSize = kRegions * kBlockSize; // Number of blocks per cacheline. @@ -235,7 +238,7 @@ crc32c_t AcceleratedCrcMemcpyEngine<vec_regions, int_regions>::Compute( const std::size_t tail_size = length - (kRegions * region_size); // Holding registers for data in each region. - std::array<__m128i, vec_regions> vec_data; + std::array<V128, vec_regions> vec_data; std::array<uint64_t, int_regions * kIntLoadsPerVec> int_data; // Main loop. @@ -243,7 +246,10 @@ crc32c_t AcceleratedCrcMemcpyEngine<vec_regions, int_regions>::Compute( // Prefetch kPrefetchAhead bytes ahead of each pointer. for (size_t i = 0; i < kRegions; i++) { absl::PrefetchToLocalCache(src_bytes + kPrefetchAhead + region_size * i); +#ifdef ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE + // TODO(b/297082454): investigate dropping prefetch on x86. absl::PrefetchToLocalCache(dst_bytes + kPrefetchAhead + region_size * i); +#endif } // Load and store data, computing CRC on the way. @@ -256,21 +262,20 @@ crc32c_t AcceleratedCrcMemcpyEngine<vec_regions, int_regions>::Compute( size_t region = (j + i) % kRegions; auto* vsrc = - reinterpret_cast<const __m128i*>(src_bytes + region_size * region); - auto* vdst = - reinterpret_cast<__m128i*>(dst_bytes + region_size * region); + reinterpret_cast<const V128*>(src_bytes + region_size * region); + auto* vdst = reinterpret_cast<V128*>(dst_bytes + region_size * region); // Load and CRC data. - vec_data[j] = _mm_loadu_si128(vsrc + i); - crcs[region] = crc32c_t{static_cast<uint32_t>(_mm_crc32_u64( - static_cast<uint32_t>(crcs[region]), - static_cast<uint64_t>(_mm_extract_epi64(vec_data[j], 0))))}; - crcs[region] = crc32c_t{static_cast<uint32_t>(_mm_crc32_u64( - static_cast<uint32_t>(crcs[region]), - static_cast<uint64_t>(_mm_extract_epi64(vec_data[j], 1))))}; + vec_data[j] = V128_LoadU(vsrc + i); + crcs[region] = crc32c_t{static_cast<uint32_t>( + CRC32_u64(static_cast<uint32_t>(crcs[region]), + static_cast<uint64_t>(V128_Extract64<0>(vec_data[j]))))}; + crcs[region] = crc32c_t{static_cast<uint32_t>( + CRC32_u64(static_cast<uint32_t>(crcs[region]), + static_cast<uint64_t>(V128_Extract64<1>(vec_data[j]))))}; // Store the data. - _mm_store_si128(vdst + i, vec_data[j]); + V128_Store(vdst + i, vec_data[j]); } // Preload the partial CRCs for the CLMUL subregions. @@ -290,7 +295,7 @@ crc32c_t AcceleratedCrcMemcpyEngine<vec_regions, int_regions>::Compute( // Load and CRC the data. int_data[data_index] = *(usrc + i * kIntLoadsPerVec + k); - crcs[region] = crc32c_t{static_cast<uint32_t>(_mm_crc32_u64( + crcs[region] = crc32c_t{static_cast<uint32_t>(CRC32_u64( static_cast<uint32_t>(crcs[region]), int_data[data_index]))}; // Store the data. @@ -313,6 +318,21 @@ crc32c_t AcceleratedCrcMemcpyEngine<vec_regions, int_regions>::Compute( src_bytes += region_size * (kRegions - 1); dst_bytes += region_size * (kRegions - 1); + // Copy and CRC the tail through the XMM registers. + std::size_t tail_blocks = tail_size / kBlockSize; + LargeTailCopy<0, 1>(&crcs[kRegions - 1], &dst_bytes, &src_bytes, 0, + tail_blocks); + + // Final tail copy for under 16 bytes. + crcs[kRegions - 1] = + ShortCrcCopy(dst_bytes, src_bytes, tail_size - tail_blocks * kBlockSize, + crcs[kRegions - 1]); + + if (kRegions == 1) { + // If there is only one region, finalize and return its CRC. + return crc32c_t{static_cast<uint32_t>(crcs[0]) ^ kCrcDataXor}; + } + // Finalize the first CRCs: XOR the internal CRCs by the XOR mask to undo the // XOR done before doing block copy + CRCs. for (size_t i = 0; i + 1 < kRegions; i++) { @@ -325,16 +345,6 @@ crc32c_t AcceleratedCrcMemcpyEngine<vec_regions, int_regions>::Compute( full_crc = ConcatCrc32c(full_crc, crcs[i], region_size); } - // Copy and CRC the tail through the XMM registers. - std::size_t tail_blocks = tail_size / kBlockSize; - LargeTailCopy<0, 1>(&crcs[kRegions - 1], &dst_bytes, &src_bytes, 0, - tail_blocks); - - // Final tail copy for under 16 bytes. - crcs[kRegions - 1] = - ShortCrcCopy(dst_bytes, src_bytes, tail_size - tail_blocks * kBlockSize, - crcs[kRegions - 1]); - // Finalize and concatenate the final CRC, then return. crcs[kRegions - 1] = crc32c_t{static_cast<uint32_t>(crcs[kRegions - 1]) ^ kCrcDataXor}; @@ -347,9 +357,11 @@ CrcMemcpy::ArchSpecificEngines CrcMemcpy::GetArchSpecificEngines() { // Get the underlying architecture. CpuType cpu_type = GetCpuType(); switch (cpu_type) { - case CpuType::kUnknown: case CpuType::kAmdRome: case CpuType::kAmdNaples: + case CpuType::kAmdMilan: + case CpuType::kAmdGenoa: + case CpuType::kAmdRyzenV3000: case CpuType::kIntelCascadelakeXeon: case CpuType::kIntelSkylakeXeon: case CpuType::kIntelSkylake: @@ -385,6 +397,9 @@ CrcMemcpy::ArchSpecificEngines CrcMemcpy::GetArchSpecificEngines() { // strided access to each region, and do the right thing. case CpuType::kAmdRome: case CpuType::kAmdNaples: + case CpuType::kAmdMilan: + case CpuType::kAmdGenoa: + case CpuType::kAmdRyzenV3000: return { /*.temporal=*/new AcceleratedCrcMemcpyEngine<1, 2>(), /*.non_temporal=*/new CrcNonTemporalMemcpyAVXEngine(), @@ -421,6 +436,8 @@ std::unique_ptr<CrcMemcpyEngine> CrcMemcpy::GetTestEngine(int vector, return std::make_unique<AcceleratedCrcMemcpyEngine<3, 0>>(); } else if (vector == 1 && integer == 2) { return std::make_unique<AcceleratedCrcMemcpyEngine<1, 2>>(); + } else if (vector == 1 && integer == 0) { + return std::make_unique<AcceleratedCrcMemcpyEngine<1, 0>>(); } return nullptr; } @@ -429,4 +446,5 @@ std::unique_ptr<CrcMemcpyEngine> CrcMemcpy::GetTestEngine(int vector, ABSL_NAMESPACE_END } // namespace absl -#endif // ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE +#endif // ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE || + // ABSL_INTERNAL_HAVE_ARM_ACCELERATED_CRC_MEMCPY_ENGINE diff --git a/contrib/restricted/abseil-cpp/absl/crc/internal/crc_x86_arm_combined.cc b/contrib/restricted/abseil-cpp/absl/crc/internal/crc_x86_arm_combined.cc index ef521d22d1..51eff4eddc 100644 --- a/contrib/restricted/abseil-cpp/absl/crc/internal/crc_x86_arm_combined.cc +++ b/contrib/restricted/abseil-cpp/absl/crc/internal/crc_x86_arm_combined.cc @@ -16,14 +16,14 @@ #include <cstddef> #include <cstdint> +#include <memory> +#include <vector> #include "absl/base/attributes.h" #include "absl/base/config.h" -#include "absl/base/dynamic_annotations.h" #include "absl/base/internal/endian.h" #include "absl/base/prefetch.h" #include "absl/crc/internal/cpu_detect.h" -#include "absl/crc/internal/crc.h" #include "absl/crc/internal/crc32_x86_arm_combined_simd.h" #include "absl/crc/internal/crc_internal.h" #include "absl/memory/memory.h" @@ -634,8 +634,16 @@ CRCImpl* TryNewCRC32AcceleratedX86ARMCombined() { return new CRC32AcceleratedX86ARMCombinedMultipleStreams< 3, 0, CutoffStrategy::Fold3>(); case CpuType::kArmNeoverseN1: + case CpuType::kArmNeoverseN2: + case CpuType::kArmNeoverseV1: return new CRC32AcceleratedX86ARMCombinedMultipleStreams< 1, 1, CutoffStrategy::Unroll64CRC>(); + case CpuType::kAmpereSiryn: + return new CRC32AcceleratedX86ARMCombinedMultipleStreams< + 3, 2, CutoffStrategy::Fold3>(); + case CpuType::kArmNeoverseV2: + return new CRC32AcceleratedX86ARMCombinedMultipleStreams< + 1, 2, CutoffStrategy::Unroll64CRC>(); #if defined(__aarch64__) default: // Not all ARM processors support the needed instructions, so check here diff --git a/contrib/restricted/abseil-cpp/absl/debugging/failure_signal_handler.cc b/contrib/restricted/abseil-cpp/absl/debugging/failure_signal_handler.cc index 992c89c390..570d1e504d 100644 --- a/contrib/restricted/abseil-cpp/absl/debugging/failure_signal_handler.cc +++ b/contrib/restricted/abseil-cpp/absl/debugging/failure_signal_handler.cc @@ -54,7 +54,7 @@ #include "absl/debugging/internal/examine_stack.h" #include "absl/debugging/stacktrace.h" -#ifndef _WIN32 +#if !defined(_WIN32) && !defined(__wasi__) #define ABSL_HAVE_SIGACTION // Apple WatchOS and TVOS don't allow sigaltstack // Apple macOS has sigaltstack, but using it makes backtrace() unusable. diff --git a/contrib/restricted/abseil-cpp/absl/debugging/internal/address_is_readable.cc b/contrib/restricted/abseil-cpp/absl/debugging/internal/address_is_readable.cc index 91eaa76f8a..be17a10591 100644 --- a/contrib/restricted/abseil-cpp/absl/debugging/internal/address_is_readable.cc +++ b/contrib/restricted/abseil-cpp/absl/debugging/internal/address_is_readable.cc @@ -50,8 +50,10 @@ namespace debugging_internal { // NOTE: any new system calls here may also require sandbox reconfiguration. // bool AddressIsReadable(const void *addr) { - // Align address on 8-byte boundary. On aarch64, checking last - // byte before inaccessible page returned unexpected EFAULT. + // rt_sigprocmask below checks 8 contiguous bytes. If addr resides in the + // last 7 bytes of a page (unaligned), rt_sigprocmask would additionally + // check the readability of the next page, which is not desired. Align + // address on 8-byte boundary to check only the current page. const uintptr_t u_addr = reinterpret_cast<uintptr_t>(addr) & ~uintptr_t{7}; addr = reinterpret_cast<const void *>(u_addr); diff --git a/contrib/restricted/abseil-cpp/absl/debugging/internal/demangle.cc b/contrib/restricted/abseil-cpp/absl/debugging/internal/demangle.cc index f2832915bf..381a2b504b 100644 --- a/contrib/restricted/abseil-cpp/absl/debugging/internal/demangle.cc +++ b/contrib/restricted/abseil-cpp/absl/debugging/internal/demangle.cc @@ -21,7 +21,15 @@ #include <cstdint> #include <cstdio> +#include <cstdlib> #include <limits> +#include <string> + +#include "absl/base/config.h" + +#if ABSL_INTERNAL_HAS_CXA_DEMANGLE +#include <cxxabi.h> +#endif namespace absl { ABSL_NAMESPACE_BEGIN @@ -1983,6 +1991,22 @@ bool Demangle(const char* mangled, char* out, size_t out_size) { state.parse_state.out_cur_idx > 0; } +std::string DemangleString(const char* mangled) { + std::string out; + int status = 0; + char* demangled = nullptr; +#if ABSL_INTERNAL_HAS_CXA_DEMANGLE + demangled = abi::__cxa_demangle(mangled, nullptr, nullptr, &status); +#endif + if (status == 0 && demangled != nullptr) { + out.append(demangled); + free(demangled); + } else { + out.append(mangled); + } + return out; +} + } // namespace debugging_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/contrib/restricted/abseil-cpp/absl/debugging/internal/demangle.h b/contrib/restricted/abseil-cpp/absl/debugging/internal/demangle.h index e1f156989f..146d1150ef 100644 --- a/contrib/restricted/abseil-cpp/absl/debugging/internal/demangle.h +++ b/contrib/restricted/abseil-cpp/absl/debugging/internal/demangle.h @@ -12,13 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -// An async-signal-safe and thread-safe demangler for Itanium C++ ABI -// (aka G++ V3 ABI). +#ifndef ABSL_DEBUGGING_INTERNAL_DEMANGLE_H_ +#define ABSL_DEBUGGING_INTERNAL_DEMANGLE_H_ + +#include <string> +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { + +// Demangle `mangled`. On success, return true and write the +// demangled symbol name to `out`. Otherwise, return false. +// `out` is modified even if demangling is unsuccessful. // -// The demangler is implemented to be used in async signal handlers to -// symbolize stack traces. We cannot use libstdc++'s -// abi::__cxa_demangle() in such signal handlers since it's not async -// signal safe (it uses malloc() internally). +// This function provides an alternative to libstdc++'s abi::__cxa_demangle, +// which is not async signal safe (it uses malloc internally). It's intended to +// be used in async signal handlers to symbolize stack traces. // // Note that this demangler doesn't support full demangling. More // specifically, it doesn't print types of function parameters and @@ -30,40 +40,32 @@ // // Example: // -// | Mangled Name | The Demangler | abi::__cxa_demangle() -// |---------------|---------------|----------------------- -// | _Z1fv | f() | f() -// | _Z1fi | f() | f(int) -// | _Z3foo3bar | foo() | foo(bar) -// | _Z1fIiEvi | f<>() | void f<int>(int) -// | _ZN1N1fE | N::f | N::f -// | _ZN3Foo3BarEv | Foo::Bar() | Foo::Bar() -// | _Zrm1XS_" | operator%() | operator%(X, X) -// | _ZN3FooC1Ev | Foo::Foo() | Foo::Foo() -// | _Z1fSs | f() | f(std::basic_string<char, -// | | | std::char_traits<char>, -// | | | std::allocator<char> >) +// | Mangled Name | Demangle | DemangleString +// |---------------|-------------|----------------------- +// | _Z1fv | f() | f() +// | _Z1fi | f() | f(int) +// | _Z3foo3bar | foo() | foo(bar) +// | _Z1fIiEvi | f<>() | void f<int>(int) +// | _ZN1N1fE | N::f | N::f +// | _ZN3Foo3BarEv | Foo::Bar() | Foo::Bar() +// | _Zrm1XS_" | operator%() | operator%(X, X) +// | _ZN3FooC1Ev | Foo::Foo() | Foo::Foo() +// | _Z1fSs | f() | f(std::basic_string<char, +// | | | std::char_traits<char>, +// | | | std::allocator<char> >) // // See the unit test for more examples. // // Note: we might want to write demanglers for ABIs other than Itanium // C++ ABI in the future. -// - -#ifndef ABSL_DEBUGGING_INTERNAL_DEMANGLE_H_ -#define ABSL_DEBUGGING_INTERNAL_DEMANGLE_H_ - -#include "absl/base/config.h" - -namespace absl { -ABSL_NAMESPACE_BEGIN -namespace debugging_internal { - -// Demangle `mangled`. On success, return true and write the -// demangled symbol name to `out`. Otherwise, return false. -// `out` is modified even if demangling is unsuccessful. bool Demangle(const char* mangled, char* out, size_t out_size); +// A wrapper around `abi::__cxa_demangle()`. On success, returns the demangled +// name. On failure, returns the input mangled name. +// +// This function is not async-signal-safe. +std::string DemangleString(const char* mangled); + } // namespace debugging_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/contrib/restricted/abseil-cpp/absl/debugging/internal/stacktrace_aarch64-inl.inc b/contrib/restricted/abseil-cpp/absl/debugging/internal/stacktrace_aarch64-inl.inc index 3f08716202..1caf7bbe35 100644 --- a/contrib/restricted/abseil-cpp/absl/debugging/internal/stacktrace_aarch64-inl.inc +++ b/contrib/restricted/abseil-cpp/absl/debugging/internal/stacktrace_aarch64-inl.inc @@ -4,6 +4,7 @@ // Generate stack tracer for aarch64 #if defined(__linux__) +#include <signal.h> #include <sys/mman.h> #include <ucontext.h> #include <unistd.h> @@ -70,7 +71,7 @@ static const unsigned char* GetKernelRtSigreturnAddress() { // Compute the size of a stack frame in [low..high). We assume that // low < high. Return size of kUnknownFrameSize. template<typename T> -static inline size_t ComputeStackFrameSize(const T* low, +static size_t ComputeStackFrameSize(const T* low, const T* high) { const char* low_char_ptr = reinterpret_cast<const char *>(low); const char* high_char_ptr = reinterpret_cast<const char *>(high); @@ -78,6 +79,20 @@ static inline size_t ComputeStackFrameSize(const T* low, : kUnknownFrameSize; } +// Saves stack info that is expensive to calculate to avoid recalculating per frame. +struct StackInfo { + uintptr_t stack_low; + uintptr_t stack_high; + uintptr_t sig_stack_low; + uintptr_t sig_stack_high; +}; + +static bool InsideSignalStack(void** ptr, const StackInfo* stack_info) { + uintptr_t comparable_ptr = reinterpret_cast<uintptr_t>(ptr); + return (comparable_ptr >= stack_info->sig_stack_low && + comparable_ptr < stack_info->sig_stack_high); +} + // Given a pointer to a stack frame, locate and return the calling // stackframe, or return null if no stackframe can be found. Perform sanity // checks (the strictness of which is controlled by the boolean parameter @@ -86,9 +101,8 @@ template<bool STRICT_UNWINDING, bool WITH_CONTEXT> ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS // May read random elements from stack. ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. static void **NextStackFrame(void **old_frame_pointer, const void *uc, - size_t stack_low, size_t stack_high) { + const StackInfo *stack_info) { void **new_frame_pointer = reinterpret_cast<void**>(*old_frame_pointer); - bool check_frame_size = true; #if defined(__linux__) if (WITH_CONTEXT && uc != nullptr) { @@ -114,10 +128,6 @@ static void **NextStackFrame(void **old_frame_pointer, const void *uc, if (!absl::debugging_internal::AddressIsReadable( new_frame_pointer)) return nullptr; - - // Skip frame size check if we return from a signal. We may be using a - // an alternate stack for signals. - check_frame_size = false; } } #endif @@ -126,9 +136,11 @@ static void **NextStackFrame(void **old_frame_pointer, const void *uc, if ((reinterpret_cast<uintptr_t>(new_frame_pointer) & 7) != 0) return nullptr; - // Check frame size. In strict mode, we assume frames to be under - // 100,000 bytes. In non-strict mode, we relax the limit to 1MB. - if (check_frame_size) { + // Only check the size if both frames are in the same stack. + if (InsideSignalStack(new_frame_pointer, stack_info) == + InsideSignalStack(old_frame_pointer, stack_info)) { + // Check frame size. In strict mode, we assume frames to be under + // 100,000 bytes. In non-strict mode, we relax the limit to 1MB. const size_t max_size = STRICT_UNWINDING ? 100000 : 1000000; const size_t frame_size = ComputeStackFrameSize(old_frame_pointer, new_frame_pointer); @@ -136,15 +148,21 @@ static void **NextStackFrame(void **old_frame_pointer, const void *uc, return nullptr; // A very large frame may mean corrupt memory or an erroneous frame // pointer. But also maybe just a plain-old large frame. Assume that if the - // frame is within the known stack, then it is valid. + // frame is within a known stack, then it is valid. if (frame_size > max_size) { - if (stack_high < kUnknownStackEnd && + size_t stack_low = stack_info->stack_low; + size_t stack_high = stack_info->stack_high; + if (InsideSignalStack(new_frame_pointer, stack_info)) { + stack_low = stack_info->sig_stack_low; + stack_high = stack_info->sig_stack_high; + } + if (stack_high < kUnknownStackEnd && static_cast<size_t>(getpagesize()) < stack_low) { const uintptr_t new_fp_u = reinterpret_cast<uintptr_t>(new_frame_pointer); // Stack bounds are known. if (!(stack_low < new_fp_u && new_fp_u <= stack_high)) { - // new_frame_pointer is not within the known stack. + // new_frame_pointer is not within a known stack. return nullptr; } } else { @@ -158,6 +176,9 @@ static void **NextStackFrame(void **old_frame_pointer, const void *uc, } template <bool IS_STACK_FRAMES, bool IS_WITH_CONTEXT> +// We count on the bottom frame being this one. See the comment +// at prev_return_address +ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS // May read random elements from stack. ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, @@ -171,8 +192,11 @@ static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, int n = 0; // Assume that the first page is not stack. - size_t stack_low = static_cast<size_t>(getpagesize()); - size_t stack_high = kUnknownStackEnd; + StackInfo stack_info; + stack_info.stack_low = static_cast<uintptr_t>(getpagesize()); + stack_info.stack_high = kUnknownStackEnd; + stack_info.sig_stack_low = stack_info.stack_low; + stack_info.sig_stack_high = kUnknownStackEnd; // The frame pointer points to low address of a frame. The first 64-bit // word of a frame points to the next frame up the call chain, which normally @@ -207,7 +231,7 @@ static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, // that is as complete as possible (even if it contains a few bogus // entries in some rare cases). frame_pointer = NextStackFrame<!IS_STACK_FRAMES, IS_WITH_CONTEXT>( - frame_pointer, ucp, stack_low, stack_high); + frame_pointer, ucp, &stack_info); } if (min_dropped_frames != nullptr) { @@ -222,7 +246,7 @@ static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, num_dropped_frames++; } frame_pointer = NextStackFrame<!IS_STACK_FRAMES, IS_WITH_CONTEXT>( - frame_pointer, ucp, stack_low, stack_high); + frame_pointer, ucp, &stack_info); } *min_dropped_frames = num_dropped_frames; } diff --git a/contrib/restricted/abseil-cpp/absl/debugging/symbolize_elf.inc b/contrib/restricted/abseil-cpp/absl/debugging/symbolize_elf.inc index 30638cb2dc..ae75cd4153 100644 --- a/contrib/restricted/abseil-cpp/absl/debugging/symbolize_elf.inc +++ b/contrib/restricted/abseil-cpp/absl/debugging/symbolize_elf.inc @@ -289,6 +289,30 @@ ObjFile *AddrMap::Add() { return new (&obj_[size_++]) ObjFile; } +class CachingFile { + public: + // Setup reader for fd that uses buf[0, buf_size-1] as a cache. + CachingFile(int fd, char *buf, size_t buf_size) + : fd_(fd), + cache_(buf), + cache_size_(buf_size), + cache_start_(0), + cache_limit_(0) {} + + int fd() const { return fd_; } + ssize_t ReadFromOffset(void *buf, size_t count, off_t offset); + bool ReadFromOffsetExact(void *buf, size_t count, off_t offset); + + private: + // Bytes [cache_start_, cache_limit_-1] from fd_ are stored in + // a prefix of cache_[0, cache_size_-1]. + int fd_; + char *cache_; + size_t cache_size_; + off_t cache_start_; + off_t cache_limit_; +}; + // --------------------------------------------------------------- enum FindSymbolResult { SYMBOL_NOT_FOUND = 1, SYMBOL_TRUNCATED, SYMBOL_FOUND }; @@ -330,6 +354,7 @@ class Symbolizer { SYMBOL_BUF_SIZE = 3072, TMP_BUF_SIZE = 1024, SYMBOL_CACHE_LINES = 128, + FILE_CACHE_SIZE = 8192, }; AddrMap addr_map_; @@ -338,6 +363,7 @@ class Symbolizer { bool addr_map_read_; char symbol_buf_[SYMBOL_BUF_SIZE]; + char file_cache_[FILE_CACHE_SIZE]; // tmp_buf_ will be used to store arrays of ElfW(Shdr) and ElfW(Sym) // so we ensure that tmp_buf_ is properly aligned to store either. @@ -436,34 +462,58 @@ static ssize_t ReadPersistent(int fd, void *buf, size_t count) { return static_cast<ssize_t>(num_bytes); } -// Read up to "count" bytes from "offset" in the file pointed by file -// descriptor "fd" into the buffer starting at "buf". On success, -// return the number of bytes read. Otherwise, return -1. -static ssize_t ReadFromOffset(const int fd, void *buf, const size_t count, - const off_t offset) { - off_t off = lseek(fd, offset, SEEK_SET); - if (off == (off_t)-1) { - ABSL_RAW_LOG(WARNING, "lseek(%d, %jd, SEEK_SET) failed: errno=%d", fd, - static_cast<intmax_t>(offset), errno); - return -1; +// Read up to "count" bytes from "offset" into the buffer starting at "buf", +// while handling short reads and EINTR. On success, return the number of bytes +// read. Otherwise, return -1. +ssize_t CachingFile::ReadFromOffset(void *buf, size_t count, off_t offset) { + char *dst = static_cast<char *>(buf); + size_t read = 0; + while (read < count) { + // Look in cache first. + if (offset >= cache_start_ && offset < cache_limit_) { + const char *hit_start = &cache_[offset - cache_start_]; + const size_t n = + std::min(count - read, static_cast<size_t>(cache_limit_ - offset)); + memcpy(dst, hit_start, n); + dst += n; + read += static_cast<size_t>(n); + offset += static_cast<off_t>(n); + continue; + } + + cache_start_ = 0; + cache_limit_ = 0; + ssize_t n = pread(fd_, cache_, cache_size_, offset); + if (n < 0) { + if (errno == EINTR) { + continue; + } + ABSL_RAW_LOG(WARNING, "read failed: errno=%d", errno); + return -1; + } + if (n == 0) { // Reached EOF. + break; + } + + cache_start_ = offset; + cache_limit_ = offset + static_cast<off_t>(n); + // Next iteration will copy from cache into dst. } - return ReadPersistent(fd, buf, count); + return static_cast<ssize_t>(read); } -// Try reading exactly "count" bytes from "offset" bytes in a file -// pointed by "fd" into the buffer starting at "buf" while handling -// short reads and EINTR. On success, return true. Otherwise, return -// false. -static bool ReadFromOffsetExact(const int fd, void *buf, const size_t count, - const off_t offset) { - ssize_t len = ReadFromOffset(fd, buf, count, offset); +// Try reading exactly "count" bytes from "offset" bytes into the buffer +// starting at "buf" while handling short reads and EINTR. On success, return +// true. Otherwise, return false. +bool CachingFile::ReadFromOffsetExact(void *buf, size_t count, off_t offset) { + ssize_t len = ReadFromOffset(buf, count, offset); return len >= 0 && static_cast<size_t>(len) == count; } // Returns elf_header.e_type if the file pointed by fd is an ELF binary. -static int FileGetElfType(const int fd) { +static int FileGetElfType(CachingFile *file) { ElfW(Ehdr) elf_header; - if (!ReadFromOffsetExact(fd, &elf_header, sizeof(elf_header), 0)) { + if (!file->ReadFromOffsetExact(&elf_header, sizeof(elf_header), 0)) { return -1; } if (memcmp(elf_header.e_ident, ELFMAG, SELFMAG) != 0) { @@ -478,8 +528,8 @@ static int FileGetElfType(const int fd) { // To keep stack consumption low, we would like this function to not get // inlined. static ABSL_ATTRIBUTE_NOINLINE bool GetSectionHeaderByType( - const int fd, ElfW(Half) sh_num, const off_t sh_offset, ElfW(Word) type, - ElfW(Shdr) * out, char *tmp_buf, size_t tmp_buf_size) { + CachingFile *file, ElfW(Half) sh_num, const off_t sh_offset, + ElfW(Word) type, ElfW(Shdr) * out, char *tmp_buf, size_t tmp_buf_size) { ElfW(Shdr) *buf = reinterpret_cast<ElfW(Shdr) *>(tmp_buf); const size_t buf_entries = tmp_buf_size / sizeof(buf[0]); const size_t buf_bytes = buf_entries * sizeof(buf[0]); @@ -490,7 +540,7 @@ static ABSL_ATTRIBUTE_NOINLINE bool GetSectionHeaderByType( const size_t num_bytes_to_read = (buf_bytes > num_bytes_left) ? num_bytes_left : buf_bytes; const off_t offset = sh_offset + static_cast<off_t>(i * sizeof(buf[0])); - const ssize_t len = ReadFromOffset(fd, buf, num_bytes_to_read, offset); + const ssize_t len = file->ReadFromOffset(buf, num_bytes_to_read, offset); if (len < 0) { ABSL_RAW_LOG( WARNING, @@ -524,11 +574,17 @@ static ABSL_ATTRIBUTE_NOINLINE bool GetSectionHeaderByType( // but there has (as yet) been no need for anything longer either. const int kMaxSectionNameLen = 64; +// Small cache to use for miscellaneous file reads. +const int kSmallFileCacheSize = 100; + bool ForEachSection(int fd, const std::function<bool(absl::string_view name, const ElfW(Shdr) &)> &callback) { + char buf[kSmallFileCacheSize]; + CachingFile file(fd, buf, sizeof(buf)); + ElfW(Ehdr) elf_header; - if (!ReadFromOffsetExact(fd, &elf_header, sizeof(elf_header), 0)) { + if (!file.ReadFromOffsetExact(&elf_header, sizeof(elf_header), 0)) { return false; } @@ -540,7 +596,7 @@ bool ForEachSection(int fd, ElfW(Shdr) shstrtab; off_t shstrtab_offset = static_cast<off_t>(elf_header.e_shoff) + elf_header.e_shentsize * elf_header.e_shstrndx; - if (!ReadFromOffsetExact(fd, &shstrtab, sizeof(shstrtab), shstrtab_offset)) { + if (!file.ReadFromOffsetExact(&shstrtab, sizeof(shstrtab), shstrtab_offset)) { return false; } @@ -548,13 +604,13 @@ bool ForEachSection(int fd, ElfW(Shdr) out; off_t section_header_offset = static_cast<off_t>(elf_header.e_shoff) + elf_header.e_shentsize * i; - if (!ReadFromOffsetExact(fd, &out, sizeof(out), section_header_offset)) { + if (!file.ReadFromOffsetExact(&out, sizeof(out), section_header_offset)) { return false; } off_t name_offset = static_cast<off_t>(shstrtab.sh_offset) + out.sh_name; char header_name[kMaxSectionNameLen]; ssize_t n_read = - ReadFromOffset(fd, &header_name, kMaxSectionNameLen, name_offset); + file.ReadFromOffset(&header_name, kMaxSectionNameLen, name_offset); if (n_read < 0) { return false; } else if (n_read > kMaxSectionNameLen) { @@ -584,8 +640,10 @@ bool GetSectionHeaderByName(int fd, const char *name, size_t name_len, return false; } + char buf[kSmallFileCacheSize]; + CachingFile file(fd, buf, sizeof(buf)); ElfW(Ehdr) elf_header; - if (!ReadFromOffsetExact(fd, &elf_header, sizeof(elf_header), 0)) { + if (!file.ReadFromOffsetExact(&elf_header, sizeof(elf_header), 0)) { return false; } @@ -597,18 +655,18 @@ bool GetSectionHeaderByName(int fd, const char *name, size_t name_len, ElfW(Shdr) shstrtab; off_t shstrtab_offset = static_cast<off_t>(elf_header.e_shoff) + elf_header.e_shentsize * elf_header.e_shstrndx; - if (!ReadFromOffsetExact(fd, &shstrtab, sizeof(shstrtab), shstrtab_offset)) { + if (!file.ReadFromOffsetExact(&shstrtab, sizeof(shstrtab), shstrtab_offset)) { return false; } for (int i = 0; i < elf_header.e_shnum; ++i) { off_t section_header_offset = static_cast<off_t>(elf_header.e_shoff) + elf_header.e_shentsize * i; - if (!ReadFromOffsetExact(fd, out, sizeof(*out), section_header_offset)) { + if (!file.ReadFromOffsetExact(out, sizeof(*out), section_header_offset)) { return false; } off_t name_offset = static_cast<off_t>(shstrtab.sh_offset) + out->sh_name; - ssize_t n_read = ReadFromOffset(fd, &header_name, name_len, name_offset); + ssize_t n_read = file.ReadFromOffset(&header_name, name_len, name_offset); if (n_read < 0) { return false; } else if (static_cast<size_t>(n_read) != name_len) { @@ -683,7 +741,7 @@ static const char *ComputeOffset(const char *base, ptrdiff_t offset) { // To keep stack consumption low, we would like this function to not get // inlined. static ABSL_ATTRIBUTE_NOINLINE FindSymbolResult FindSymbol( - const void *const pc, const int fd, char *out, size_t out_size, + const void *const pc, CachingFile *file, char *out, size_t out_size, ptrdiff_t relocation, const ElfW(Shdr) * strtab, const ElfW(Shdr) * symtab, const ElfW(Shdr) * opd, char *tmp_buf, size_t tmp_buf_size) { if (symtab == nullptr) { @@ -716,7 +774,7 @@ static ABSL_ATTRIBUTE_NOINLINE FindSymbolResult FindSymbol( const size_t entries_in_chunk = std::min(num_remaining_symbols, buf_entries); const size_t bytes_in_chunk = entries_in_chunk * sizeof(buf[0]); - const ssize_t len = ReadFromOffset(fd, buf, bytes_in_chunk, offset); + const ssize_t len = file->ReadFromOffset(buf, bytes_in_chunk, offset); SAFE_ASSERT(len >= 0); SAFE_ASSERT(static_cast<size_t>(len) % sizeof(buf[0]) == 0); const size_t num_symbols_in_buf = static_cast<size_t>(len) / sizeof(buf[0]); @@ -772,12 +830,12 @@ static ABSL_ATTRIBUTE_NOINLINE FindSymbolResult FindSymbol( if (found_match) { const off_t off = static_cast<off_t>(strtab->sh_offset) + best_match.st_name; - const ssize_t n_read = ReadFromOffset(fd, out, out_size, off); + const ssize_t n_read = file->ReadFromOffset(out, out_size, off); if (n_read <= 0) { // This should never happen. ABSL_RAW_LOG(WARNING, - "Unable to read from fd %d at offset %lld: n_read = %zd", fd, - static_cast<long long>(off), n_read); + "Unable to read from fd %d at offset %lld: n_read = %zd", + file->fd(), static_cast<long long>(off), n_read); return SYMBOL_NOT_FOUND; } ABSL_RAW_CHECK(static_cast<size_t>(n_read) <= out_size, @@ -827,22 +885,24 @@ FindSymbolResult Symbolizer::GetSymbolFromObjectFile( } } + CachingFile file(obj.fd, file_cache_, sizeof(file_cache_)); + // Consult a regular symbol table, then fall back to the dynamic symbol table. for (const auto symbol_table_type : {SHT_SYMTAB, SHT_DYNSYM}) { - if (!GetSectionHeaderByType(obj.fd, obj.elf_header.e_shnum, + if (!GetSectionHeaderByType(&file, obj.elf_header.e_shnum, static_cast<off_t>(obj.elf_header.e_shoff), static_cast<ElfW(Word)>(symbol_table_type), &symtab, tmp_buf, tmp_buf_size)) { continue; } - if (!ReadFromOffsetExact( - obj.fd, &strtab, sizeof(strtab), + if (!file.ReadFromOffsetExact( + &strtab, sizeof(strtab), static_cast<off_t>(obj.elf_header.e_shoff + symtab.sh_link * sizeof(symtab)))) { continue; } const FindSymbolResult rc = - FindSymbol(pc, obj.fd, out, out_size, relocation, &strtab, &symtab, + FindSymbol(pc, &file, out, out_size, relocation, &strtab, &symtab, opd_ptr, tmp_buf, tmp_buf_size); if (rc != SYMBOL_NOT_FOUND) { return rc; @@ -1323,15 +1383,19 @@ static bool MaybeInitializeObjFile(ObjFile *obj) { ABSL_RAW_LOG(WARNING, "%s: open failed: errno=%d", obj->filename, errno); return false; } - obj->elf_type = FileGetElfType(obj->fd); + + char buf[kSmallFileCacheSize]; + CachingFile file(obj->fd, buf, sizeof(buf)); + + obj->elf_type = FileGetElfType(&file); if (obj->elf_type < 0) { ABSL_RAW_LOG(WARNING, "%s: wrong elf type: %d", obj->filename, obj->elf_type); return false; } - if (!ReadFromOffsetExact(obj->fd, &obj->elf_header, sizeof(obj->elf_header), - 0)) { + if (!file.ReadFromOffsetExact(&obj->elf_header, sizeof(obj->elf_header), + 0)) { ABSL_RAW_LOG(WARNING, "%s: failed to read elf header", obj->filename); return false; } @@ -1341,7 +1405,7 @@ static bool MaybeInitializeObjFile(ObjFile *obj) { size_t num_interesting_load_segments = 0; for (int j = 0; j < phnum; j++) { ElfW(Phdr) phdr; - if (!ReadFromOffsetExact(obj->fd, &phdr, sizeof(phdr), phoff)) { + if (!file.ReadFromOffsetExact(&phdr, sizeof(phdr), phoff)) { ABSL_RAW_LOG(WARNING, "%s: failed to read program header %d", obj->filename, j); return false; diff --git a/contrib/restricted/abseil-cpp/absl/flags/declare.h b/contrib/restricted/abseil-cpp/absl/flags/declare.h index d1437bb9f6..8d2a856e3b 100644 --- a/contrib/restricted/abseil-cpp/absl/flags/declare.h +++ b/contrib/restricted/abseil-cpp/absl/flags/declare.h @@ -40,13 +40,8 @@ class Flag; // Flag // // Forward declaration of the `absl::Flag` type for use in defining the macro. -#if defined(_MSC_VER) && !defined(__clang__) -template <typename T> -class Flag; -#else template <typename T> using Flag = flags_internal::Flag<T>; -#endif ABSL_NAMESPACE_END } // namespace absl diff --git a/contrib/restricted/abseil-cpp/absl/flags/flag.h b/contrib/restricted/abseil-cpp/absl/flags/flag.h index b7f94be7c5..06ea6932f9 100644 --- a/contrib/restricted/abseil-cpp/absl/flags/flag.h +++ b/contrib/restricted/abseil-cpp/absl/flags/flag.h @@ -71,12 +71,9 @@ ABSL_NAMESPACE_BEGIN // For type support of Abseil Flags, see the marshalling.h header file, which // discusses supported standard types, optional flags, and additional Abseil // type support. -#if !defined(_MSC_VER) || defined(__clang__) + template <typename T> using Flag = flags_internal::Flag<T>; -#else -#include "absl/flags/internal/flag_msvc.inc" -#endif // GetFlag() // @@ -196,18 +193,12 @@ ABSL_NAMESPACE_END // ----------------------------------------------------------------------------- // ABSL_FLAG_IMPL macro definition conditional on ABSL_FLAGS_STRIP_NAMES -#if !defined(_MSC_VER) || defined(__clang__) #define ABSL_FLAG_IMPL_FLAG_PTR(flag) flag #define ABSL_FLAG_IMPL_HELP_ARG(name) \ absl::flags_internal::HelpArg<AbslFlagHelpGenFor##name>( \ FLAGS_help_storage_##name) #define ABSL_FLAG_IMPL_DEFAULT_ARG(Type, name) \ absl::flags_internal::DefaultArg<Type, AbslFlagDefaultGenFor##name>(0) -#else -#define ABSL_FLAG_IMPL_FLAG_PTR(flag) flag.GetImpl() -#define ABSL_FLAG_IMPL_HELP_ARG(name) &AbslFlagHelpGenFor##name::NonConst -#define ABSL_FLAG_IMPL_DEFAULT_ARG(Type, name) &AbslFlagDefaultGenFor##name::Gen -#endif #if ABSL_FLAGS_STRIP_NAMES #define ABSL_FLAG_IMPL_FLAGNAME(txt) "" diff --git a/contrib/restricted/abseil-cpp/absl/flags/internal/flag.h b/contrib/restricted/abseil-cpp/absl/flags/internal/flag.h index b41f9a69f8..2e6e6b8742 100644 --- a/contrib/restricted/abseil-cpp/absl/flags/internal/flag.h +++ b/contrib/restricted/abseil-cpp/absl/flags/internal/flag.h @@ -54,13 +54,8 @@ template <typename T> class Flag; } // namespace flags_internal -#if defined(_MSC_VER) && !defined(__clang__) -template <typename T> -class Flag; -#else template <typename T> using Flag = flags_internal::Flag<T>; -#endif template <typename T> ABSL_MUST_USE_RESULT T GetFlag(const absl::Flag<T>& flag); diff --git a/contrib/restricted/abseil-cpp/absl/flags/internal/flag_msvc.inc b/contrib/restricted/abseil-cpp/absl/flags/internal/flag_msvc.inc deleted file mode 100644 index 614d09fd1a..0000000000 --- a/contrib/restricted/abseil-cpp/absl/flags/internal/flag_msvc.inc +++ /dev/null @@ -1,116 +0,0 @@ -// -// Copyright 2021 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Do not include this file directly. -// Include absl/flags/flag.h instead. - -// MSVC debug builds do not implement initialization with constexpr constructors -// correctly. To work around this we add a level of indirection, so that the -// class `absl::Flag` contains an `internal::Flag*` (instead of being an alias -// to that class) and dynamically allocates an instance when necessary. We also -// forward all calls to internal::Flag methods via trampoline methods. In this -// setup the `absl::Flag` class does not have constructor and virtual methods, -// all the data members are public and thus MSVC is able to initialize it at -// link time. To deal with multiple threads accessing the flag for the first -// time concurrently we use an atomic boolean indicating if flag object is -// initialized. We also employ the double-checked locking pattern where the -// second level of protection is a global Mutex, so if two threads attempt to -// construct the flag concurrently only one wins. -// -// This solution is based on a recommendation here: -// https://developercommunity.visualstudio.com/content/problem/336946/class-with-constexpr-constructor-not-using-static.html?childToView=648454#comment-648454 - -namespace flags_internal { -absl::Mutex* GetGlobalConstructionGuard(); -} // namespace flags_internal - -// Public methods of `absl::Flag<T>` are NOT part of the Abseil Flags API. -// See https://abseil.io/docs/cpp/guides/flags -template <typename T> -class Flag { - public: - // No constructor and destructor to ensure this is an aggregate type. - // Visual Studio 2015 still requires the constructor for class to be - // constexpr initializable. -#if _MSC_VER <= 1900 - constexpr Flag(const char* name, const char* filename, - const flags_internal::HelpGenFunc help_gen, - const flags_internal::FlagDfltGenFunc default_value_gen) - : name_(name), - filename_(filename), - help_gen_(help_gen), - default_value_gen_(default_value_gen), - inited_(false), - impl_(nullptr) {} -#endif - - flags_internal::Flag<T>& GetImpl() const { - if (!inited_.load(std::memory_order_acquire)) { - absl::MutexLock l(flags_internal::GetGlobalConstructionGuard()); - - if (inited_.load(std::memory_order_acquire)) { - return *impl_; - } - - impl_ = new flags_internal::Flag<T>( - name_, filename_, - {flags_internal::FlagHelpMsg(help_gen_), - flags_internal::FlagHelpKind::kGenFunc}, - {flags_internal::FlagDefaultSrc(default_value_gen_), - flags_internal::FlagDefaultKind::kGenFunc}); - inited_.store(true, std::memory_order_release); - } - - return *impl_; - } - - // Public methods of `absl::Flag<T>` are NOT part of the Abseil Flags API. - // See https://abseil.io/docs/cpp/guides/flags - bool IsRetired() const { return GetImpl().IsRetired(); } - absl::string_view Name() const { return GetImpl().Name(); } - std::string Help() const { return GetImpl().Help(); } - bool IsModified() const { return GetImpl().IsModified(); } - bool IsSpecifiedOnCommandLine() const { - return GetImpl().IsSpecifiedOnCommandLine(); - } - std::string Filename() const { return GetImpl().Filename(); } - std::string DefaultValue() const { return GetImpl().DefaultValue(); } - std::string CurrentValue() const { return GetImpl().CurrentValue(); } - template <typename U> - inline bool IsOfType() const { - return GetImpl().template IsOfType<U>(); - } - T Get() const { - return flags_internal::FlagImplPeer::InvokeGet<T>(GetImpl()); - } - void Set(const T& v) { - flags_internal::FlagImplPeer::InvokeSet(GetImpl(), v); - } - void InvokeCallback() { GetImpl().InvokeCallback(); } - - const CommandLineFlag& Reflect() const { - return flags_internal::FlagImplPeer::InvokeReflect(GetImpl()); - } - - // The data members are logically private, but they need to be public for - // this to be an aggregate type. - const char* name_; - const char* filename_; - const flags_internal::HelpGenFunc help_gen_; - const flags_internal::FlagDfltGenFunc default_value_gen_; - - mutable std::atomic<bool> inited_; - mutable flags_internal::Flag<T>* impl_; -}; diff --git a/contrib/restricted/abseil-cpp/absl/flags/internal/usage.cc b/contrib/restricted/abseil-cpp/absl/flags/internal/usage.cc index 13852e1467..8b169bcdc2 100644 --- a/contrib/restricted/abseil-cpp/absl/flags/internal/usage.cc +++ b/contrib/restricted/abseil-cpp/absl/flags/internal/usage.cc @@ -27,7 +27,10 @@ #include <utility> #include <vector> +#include "absl/base/attributes.h" #include "absl/base/config.h" +#include "absl/base/const_init.h" +#include "absl/base/thread_annotations.h" #include "absl/flags/commandlineflag.h" #include "absl/flags/flag.h" #include "absl/flags/internal/flag.h" @@ -40,6 +43,8 @@ #include "absl/strings/str_cat.h" #include "absl/strings/str_split.h" #include "absl/strings/string_view.h" +#include "absl/strings/strip.h" +#include "absl/synchronization/mutex.h" // Dummy global variables to prevent anyone else defining these. bool FLAGS_help = false; diff --git a/contrib/restricted/abseil-cpp/absl/flags/marshalling.cc b/contrib/restricted/abseil-cpp/absl/flags/marshalling.cc index dc69754f00..ca4a130572 100644 --- a/contrib/restricted/abseil-cpp/absl/flags/marshalling.cc +++ b/contrib/restricted/abseil-cpp/absl/flags/marshalling.cc @@ -247,6 +247,14 @@ bool AbslParseFlag(absl::string_view text, absl::LogSeverity* dst, *err = "no value provided"; return false; } + if (absl::EqualsIgnoreCase(text, "dfatal")) { + *dst = absl::kLogDebugFatal; + return true; + } + if (absl::EqualsIgnoreCase(text, "klogdebugfatal")) { + *dst = absl::kLogDebugFatal; + return true; + } if (text.front() == 'k' || text.front() == 'K') text.remove_prefix(1); if (absl::EqualsIgnoreCase(text, "info")) { *dst = absl::LogSeverity::kInfo; @@ -269,7 +277,8 @@ bool AbslParseFlag(absl::string_view text, absl::LogSeverity* dst, *dst = static_cast<absl::LogSeverity>(numeric_value); return true; } - *err = "only integers and absl::LogSeverity enumerators are accepted"; + *err = + "only integers, absl::LogSeverity enumerators, and DFATAL are accepted"; return false; } diff --git a/contrib/restricted/abseil-cpp/absl/flags/parse.cc b/contrib/restricted/abseil-cpp/absl/flags/parse.cc index 4cdd9d0ad3..526b61d1d1 100644 --- a/contrib/restricted/abseil-cpp/absl/flags/parse.cc +++ b/contrib/restricted/abseil-cpp/absl/flags/parse.cc @@ -637,7 +637,7 @@ void ReportUnrecognizedFlags( // -------------------------------------------------------------------- bool WasPresentOnCommandLine(absl::string_view flag_name) { - absl::MutexLock l(&specified_flags_guard); + absl::ReaderMutexLock l(&specified_flags_guard); ABSL_INTERNAL_CHECK(specified_flags != nullptr, "ParseCommandLine is not invoked yet"); diff --git a/contrib/restricted/abseil-cpp/absl/flags/reflection.cc b/contrib/restricted/abseil-cpp/absl/flags/reflection.cc index dbce4032ab..841921a926 100644 --- a/contrib/restricted/abseil-cpp/absl/flags/reflection.cc +++ b/contrib/restricted/abseil-cpp/absl/flags/reflection.cc @@ -21,6 +21,7 @@ #include <string> #include "absl/base/config.h" +#include "absl/base/no_destructor.h" #include "absl/base/thread_annotations.h" #include "absl/container/flat_hash_map.h" #include "absl/flags/commandlineflag.h" @@ -169,7 +170,7 @@ void FlagRegistry::RegisterFlag(CommandLineFlag& flag, const char* filename) { } FlagRegistry& FlagRegistry::GlobalRegistry() { - static FlagRegistry* global_registry = new FlagRegistry; + static absl::NoDestructor<FlagRegistry> global_registry; return *global_registry; } diff --git a/contrib/restricted/abseil-cpp/absl/flags/ya.make b/contrib/restricted/abseil-cpp/absl/flags/ya.make index f0006d104f..7e6fae43d6 100644 --- a/contrib/restricted/abseil-cpp/absl/flags/ya.make +++ b/contrib/restricted/abseil-cpp/absl/flags/ya.make @@ -30,7 +30,6 @@ NO_UTIL() SRCS( commandlineflag.cc - flag.cc internal/commandlineflag.cc internal/flag.cc internal/private_handle_accessor.cc diff --git a/contrib/restricted/abseil-cpp/absl/functional/function_ref.h b/contrib/restricted/abseil-cpp/absl/functional/function_ref.h index 2b9139d374..96cece551b 100644 --- a/contrib/restricted/abseil-cpp/absl/functional/function_ref.h +++ b/contrib/restricted/abseil-cpp/absl/functional/function_ref.h @@ -137,6 +137,14 @@ class FunctionRef<R(Args...)> { absl::functional_internal::Invoker<R, Args...> invoker_; }; +// Allow const qualified function signatures. Since FunctionRef requires +// constness anyway we can just make this a no-op. +template <typename R, typename... Args> +class FunctionRef<R(Args...) const> : public FunctionRef<R(Args...)> { + public: + using FunctionRef<R(Args...)>::FunctionRef; +}; + ABSL_NAMESPACE_END } // namespace absl diff --git a/contrib/restricted/abseil-cpp/absl/functional/internal/any_invocable.h b/contrib/restricted/abseil-cpp/absl/functional/internal/any_invocable.h index f096bb02e1..b04436d1d6 100644 --- a/contrib/restricted/abseil-cpp/absl/functional/internal/any_invocable.h +++ b/contrib/restricted/abseil-cpp/absl/functional/internal/any_invocable.h @@ -215,8 +215,8 @@ T& ObjectInLocalStorage(TypeErasedState* const state) { // behavior, which works as intended on Abseil's officially supported // platforms as of Q2 2022. #if !defined(__clang__) && defined(__GNUC__) -#pragma GCC diagnostic ignored "-Wstrict-aliasing" #pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wstrict-aliasing" #endif return *reinterpret_cast<T*>(&state->storage); #if !defined(__clang__) && defined(__GNUC__) @@ -526,10 +526,10 @@ class CoreImpl { // Since this is template-heavy code, we prefer to disable these warnings // locally instead of adding yet another overload of this function. #if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wpragmas" #pragma GCC diagnostic ignored "-Waddress" #pragma GCC diagnostic ignored "-Wnonnull-compare" -#pragma GCC diagnostic push #endif if (static_cast<RemoveCVRef<QualDecayedTRef>>(f) == nullptr) { #if !defined(__clang__) && defined(__GNUC__) diff --git a/contrib/restricted/abseil-cpp/absl/functional/ya.make b/contrib/restricted/abseil-cpp/absl/functional/ya.make index 3b66e3e400..3f9af85fe6 100644 --- a/contrib/restricted/abseil-cpp/absl/functional/ya.make +++ b/contrib/restricted/abseil-cpp/absl/functional/ya.make @@ -6,9 +6,9 @@ LICENSE(Apache-2.0) LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(20230802.1) +VERSION(20240116.0) -ORIGINAL_SOURCE(https://github.com/abseil/abseil-cpp/archive/20230802.1.tar.gz) +ORIGINAL_SOURCE(https://github.com/abseil/abseil-cpp/archive/20240116.0.tar.gz) NO_RUNTIME() diff --git a/contrib/restricted/abseil-cpp/absl/hash/internal/hash.h b/contrib/restricted/abseil-cpp/absl/hash/internal/hash.h index ef3f366471..f4a94f9129 100644 --- a/contrib/restricted/abseil-cpp/absl/hash/internal/hash.h +++ b/contrib/restricted/abseil-cpp/absl/hash/internal/hash.h @@ -19,6 +19,11 @@ #ifndef ABSL_HASH_INTERNAL_HASH_H_ #define ABSL_HASH_INTERNAL_HASH_H_ +#ifdef __APPLE__ +#include <Availability.h> +#include <TargetConditionals.h> +#endif + #include <algorithm> #include <array> #include <bitset> @@ -56,6 +61,11 @@ #include "absl/types/variant.h" #include "absl/utility/utility.h" +#if ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L && \ + !defined(_LIBCPP_HAS_NO_FILESYSTEM_LIBRARY) +#include <filesystem> // NOLINT +#endif + #ifdef ABSL_HAVE_STD_STRING_VIEW #include <string_view> #endif @@ -409,9 +419,23 @@ AbslHashValue(H hash_state, LongDouble value) { return H::combine(std::move(hash_state), category); } +// Without this overload, an array decays to a pointer and we hash that, which +// is not likely to be what the caller intended. +template <typename H, typename T, size_t N> +H AbslHashValue(H hash_state, T (&)[N]) { + static_assert( + sizeof(T) == -1, + "Hashing C arrays is not allowed. For string literals, wrap the literal " + "in absl::string_view(). To hash the array contents, use " + "absl::MakeSpan() or make the array an std::array. To hash the array " + "address, use &array[0]."); + return hash_state; +} + // AbslHashValue() for hashing pointers template <typename H, typename T> -H AbslHashValue(H hash_state, T* ptr) { +std::enable_if_t<std::is_pointer<T>::value, H> AbslHashValue(H hash_state, + T ptr) { auto v = reinterpret_cast<uintptr_t>(ptr); // Due to alignment, pointers tend to have low bits as zero, and the next few // bits follow a pattern since they are also multiples of some base value. @@ -564,6 +588,29 @@ H AbslHashValue(H hash_state, std::basic_string_view<Char> str) { #endif // ABSL_HAVE_STD_STRING_VIEW +#if defined(__cpp_lib_filesystem) && __cpp_lib_filesystem >= 201703L && \ + !defined(_LIBCPP_HAS_NO_FILESYSTEM_LIBRARY) && \ + (!defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) || \ + __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ >= 130000) + +#define ABSL_INTERNAL_STD_FILESYSTEM_PATH_HASH_AVAILABLE 1 + +// Support std::filesystem::path. The SFINAE is required because some string +// types are implicitly convertible to std::filesystem::path. +template <typename Path, typename H, + typename = absl::enable_if_t< + std::is_same_v<Path, std::filesystem::path>>> +H AbslHashValue(H hash_state, const Path& path) { + // This is implemented by deferring to the standard library to compute the + // hash. The standard library requires that for two paths, `p1 == p2`, then + // `hash_value(p1) == hash_value(p2)`. `AbslHashValue` has the same + // requirement. Since `operator==` does platform specific matching, deferring + // to the standard library is the simplest approach. + return H::combine(std::move(hash_state), std::filesystem::hash_value(path)); +} + +#endif // ABSL_INTERNAL_STD_FILESYSTEM_PATH_HASH_AVAILABLE + // ----------------------------------------------------------------------------- // AbslHashValue for Sequence Containers // ----------------------------------------------------------------------------- @@ -818,7 +865,7 @@ AbslHashValue(H hash_state, const absl::variant<T...>& v) { template <typename H, size_t N> H AbslHashValue(H hash_state, const std::bitset<N>& set) { typename H::AbslInternalPiecewiseCombiner combiner; - for (int i = 0; i < N; i++) { + for (size_t i = 0; i < N; i++) { unsigned char c = static_cast<unsigned char>(set[i]); hash_state = combiner.add_buffer(std::move(hash_state), &c, sizeof(c)); } diff --git a/contrib/restricted/abseil-cpp/absl/log/absl_vlog_is_on.h b/contrib/restricted/abseil-cpp/absl/log/absl_vlog_is_on.h new file mode 100644 index 0000000000..29096b4857 --- /dev/null +++ b/contrib/restricted/abseil-cpp/absl/log/absl_vlog_is_on.h @@ -0,0 +1,93 @@ +// Copyright 2022 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: log/absl_vlog_is_on.h +// ----------------------------------------------------------------------------- +// +// This header defines the `ABSL_VLOG_IS_ON()` macro that controls the +// variable-verbosity conditional logging. +// +// It's used by `VLOG` in log.h, or it can also be used directly like this: +// +// if (ABSL_VLOG_IS_ON(2)) { +// foo_server.RecomputeStatisticsExpensive(); +// LOG(INFO) << foo_server.LastStatisticsAsString(); +// } +// +// Each source file has an effective verbosity level that's a non-negative +// integer computed from the `--vmodule` and `--v` flags. +// `ABSL_VLOG_IS_ON(n)` is true, and `VLOG(n)` logs, if that effective verbosity +// level is greater than or equal to `n`. +// +// `--vmodule` takes a comma-delimited list of key=value pairs. Each key is a +// pattern matched against filenames, and the values give the effective severity +// level applied to matching files. '?' and '*' characters in patterns are +// interpreted as single-character and zero-or-more-character wildcards. +// Patterns including a slash character are matched against full pathnames, +// while those without are matched against basenames only. One suffix (i.e. the +// last . and everything after it) is stripped from each filename prior to +// matching, as is the special suffix "-inl". +// +// Files are matched against globs in `--vmodule` in order, and the first match +// determines the verbosity level. +// +// Files which do not match any pattern in `--vmodule` use the value of `--v` as +// their effective verbosity level. The default is 0. +// +// SetVLOGLevel helper function is provided to do limited dynamic control over +// V-logging by appending to `--vmodule`. Because these go at the beginning of +// the list, they take priority over any globs previously added. +// +// Resetting --vmodule will override all previous modifications to `--vmodule`, +// including via SetVLOGLevel. + +#ifndef ABSL_LOG_ABSL_VLOG_IS_ON_H_ +#define ABSL_LOG_ABSL_VLOG_IS_ON_H_ + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/log/internal/vlog_config.h" // IWYU pragma: export +#include "absl/strings/string_view.h" + +// IWYU pragma: private, include "absl/log/log.h" + +// This is expanded at the callsite to allow the compiler to optimize +// always-false cases out of the build. +// An ABSL_MAX_VLOG_VERBOSITY of 2 means that VLOG(3) and above should never +// log. +#ifdef ABSL_MAX_VLOG_VERBOSITY +#define ABSL_LOG_INTERNAL_MAX_LOG_VERBOSITY_CHECK(x) \ + ((x) <= ABSL_MAX_VLOG_VERBOSITY)&& +#else +#define ABSL_LOG_INTERNAL_MAX_LOG_VERBOSITY_CHECK(x) +#endif + +// Each ABSL_VLOG_IS_ON call site gets its own VLogSite that registers with the +// global linked list of sites to asynchronously update its verbosity level on +// changes to --v or --vmodule. The verbosity can also be set by manually +// calling SetVLOGLevel. +// +// ABSL_VLOG_IS_ON is not async signal safe, but it is guaranteed not to +// allocate new memory. +#define ABSL_VLOG_IS_ON(verbose_level) \ + (ABSL_LOG_INTERNAL_MAX_LOG_VERBOSITY_CHECK(verbose_level)[]() \ + ->::absl::log_internal::VLogSite * \ + { \ + ABSL_CONST_INIT static ::absl::log_internal::VLogSite site(__FILE__); \ + return &site; \ + }() \ + ->IsEnabled(verbose_level)) + +#endif // ABSL_LOG_ABSL_VLOG_IS_ON_H_ diff --git a/contrib/restricted/abseil-cpp/absl/log/flags.cc b/contrib/restricted/abseil-cpp/absl/log/flags.cc index 215b7bd580..287b3e96c7 100644 --- a/contrib/restricted/abseil-cpp/absl/log/flags.cc +++ b/contrib/restricted/abseil-cpp/absl/log/flags.cc @@ -28,6 +28,7 @@ #include "absl/flags/marshalling.h" #include "absl/log/globals.h" #include "absl/log/internal/config.h" +#include "absl/log/internal/vlog_config.h" #include "absl/strings/numbers.h" #include "absl/strings/string_view.h" @@ -118,3 +119,25 @@ ABSL_FLAG(bool, log_prefix, true, .OnUpdate([] { absl::log_internal::RawEnableLogPrefix(absl::GetFlag(FLAGS_log_prefix)); }); + +ABSL_FLAG(int, v, 0, + "Show all VLOG(m) messages for m <= this. Overridable by --vmodule.") + .OnUpdate([] { + absl::log_internal::UpdateGlobalVLogLevel(absl::GetFlag(FLAGS_v)); + }); + +ABSL_FLAG( + std::string, vmodule, "", + "per-module log verbosity level." + " Argument is a comma-separated list of <module name>=<log level>." + " <module name> is a glob pattern, matched against the filename base" + " (that is, name ignoring .cc/.h./-inl.h)." + " A pattern without slashes matches just the file name portion, otherwise" + " the whole file path below the workspace root" + " (still without .cc/.h./-inl.h) is matched." + " ? and * in the glob pattern match any single or sequence of characters" + " respectively including slashes." + " <log level> overrides any value given by --v.") + .OnUpdate([] { + absl::log_internal::UpdateVModule(absl::GetFlag(FLAGS_vmodule)); + }); diff --git a/contrib/restricted/abseil-cpp/absl/log/globals.h b/contrib/restricted/abseil-cpp/absl/log/globals.h index bc3864c110..b36e47e6d1 100644 --- a/contrib/restricted/abseil-cpp/absl/log/globals.h +++ b/contrib/restricted/abseil-cpp/absl/log/globals.h @@ -24,6 +24,7 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/log_severity.h" +#include "absl/log/internal/vlog_config.h" #include "absl/strings/string_view.h" namespace absl { @@ -153,6 +154,28 @@ ABSL_MUST_USE_RESULT bool ShouldPrependLogPrefix(); void EnableLogPrefix(bool on_off); //------------------------------------------------------------------------------ +// Set Global VLOG Level +//------------------------------------------------------------------------------ +// +// Sets the global `(ABSL_)VLOG(_IS_ON)` level to `log_level`. This level is +// applied to any sites whose filename doesn't match any `module_pattern`. +// Returns the prior value. +inline int SetGlobalVLogLevel(int log_level) { + return absl::log_internal::UpdateGlobalVLogLevel(log_level); +} + +//------------------------------------------------------------------------------ +// Set VLOG Level +//------------------------------------------------------------------------------ +// +// Sets `(ABSL_)VLOG(_IS_ON)` level for `module_pattern` to `log_level`. This +// allows programmatic control of what is normally set by the --vmodule flag. +// Returns the level that previously applied to `module_pattern`. +inline int SetVLogLevel(absl::string_view module_pattern, int log_level) { + return absl::log_internal::PrependVModule(module_pattern, log_level); +} + +//------------------------------------------------------------------------------ // Configure Android Native Log Tag //------------------------------------------------------------------------------ // diff --git a/contrib/restricted/abseil-cpp/absl/log/initialize.cc b/contrib/restricted/abseil-cpp/absl/log/initialize.cc index a3f6d6c142..ef5d31469a 100644 --- a/contrib/restricted/abseil-cpp/absl/log/initialize.cc +++ b/contrib/restricted/abseil-cpp/absl/log/initialize.cc @@ -21,14 +21,18 @@ namespace absl { ABSL_NAMESPACE_BEGIN -void InitializeLog() { +namespace { +void InitializeLogImpl(absl::TimeZone time_zone) { // This comes first since it is used by RAW_LOG. - absl::log_internal::SetTimeZone(absl::LocalTimeZone()); + absl::log_internal::SetTimeZone(time_zone); // Note that initialization is complete, so logs can now be sent to their // proper destinations rather than stderr. log_internal::SetInitialized(); } +} // namespace + +void InitializeLog() { InitializeLogImpl(absl::LocalTimeZone()); } ABSL_NAMESPACE_END } // namespace absl diff --git a/contrib/restricted/abseil-cpp/absl/log/internal/check_op.h b/contrib/restricted/abseil-cpp/absl/log/internal/check_op.h index 20b01b5e84..11f0f4078b 100644 --- a/contrib/restricted/abseil-cpp/absl/log/internal/check_op.h +++ b/contrib/restricted/abseil-cpp/absl/log/internal/check_op.h @@ -65,6 +65,7 @@ ::absl::log_internal::GetReferenceableValue(val2), \ ABSL_LOG_INTERNAL_STRIP_STRING_LITERAL(val1_text \ " " #op " " val2_text))) \ + ABSL_LOG_INTERNAL_CONDITION_FATAL(STATELESS, true) \ ABSL_LOG_INTERNAL_CHECK(*absl_log_internal_check_op_result).InternalStream() #define ABSL_LOG_INTERNAL_QCHECK_OP(name, op, val1, val1_text, val2, \ val2_text) \ @@ -74,6 +75,7 @@ ::absl::log_internal::GetReferenceableValue(val2), \ ABSL_LOG_INTERNAL_STRIP_STRING_LITERAL( \ val1_text " " #op " " val2_text))) \ + ABSL_LOG_INTERNAL_CONDITION_QFATAL(STATELESS, true) \ ABSL_LOG_INTERNAL_QCHECK(*absl_log_internal_qcheck_op_result).InternalStream() #define ABSL_LOG_INTERNAL_CHECK_STROP(func, op, expected, s1, s1_text, s2, \ s2_text) \ @@ -82,6 +84,7 @@ (s1), (s2), \ ABSL_LOG_INTERNAL_STRIP_STRING_LITERAL(s1_text " " #op \ " " s2_text))) \ + ABSL_LOG_INTERNAL_CONDITION_FATAL(STATELESS, true) \ ABSL_LOG_INTERNAL_CHECK(*absl_log_internal_check_strop_result) \ .InternalStream() #define ABSL_LOG_INTERNAL_QCHECK_STROP(func, op, expected, s1, s1_text, s2, \ @@ -91,6 +94,7 @@ (s1), (s2), \ ABSL_LOG_INTERNAL_STRIP_STRING_LITERAL(s1_text " " #op \ " " s2_text))) \ + ABSL_LOG_INTERNAL_CONDITION_QFATAL(STATELESS, true) \ ABSL_LOG_INTERNAL_QCHECK(*absl_log_internal_qcheck_strop_result) \ .InternalStream() // This one is tricky: @@ -113,6 +117,10 @@ // * As usual, no braces so we can stream into the expansion with `operator<<`. // * Also as usual, it must expand to a single (partial) statement with no // ambiguous-else problems. +// * When stripped by `ABSL_MIN_LOG_LEVEL`, we must discard the `<expr> is OK` +// string literal and abort without doing any streaming. We don't need to +// strip the call to stringify the non-ok `Status` as long as we don't log it; +// dropping the `Status`'s message text is out of scope. #define ABSL_LOG_INTERNAL_CHECK_OK(val, val_text) \ for (::std::pair<const ::absl::Status*, ::std::string*> \ absl_log_internal_check_ok_goo; \ @@ -126,22 +134,24 @@ ABSL_LOG_INTERNAL_STRIP_STRING_LITERAL(val_text \ " is OK")), \ !ABSL_PREDICT_TRUE(absl_log_internal_check_ok_goo.first->ok());) \ + ABSL_LOG_INTERNAL_CONDITION_FATAL(STATELESS, true) \ ABSL_LOG_INTERNAL_CHECK(*absl_log_internal_check_ok_goo.second) \ .InternalStream() -#define ABSL_LOG_INTERNAL_QCHECK_OK(val, val_text) \ - for (::std::pair<const ::absl::Status*, ::std::string*> \ - absl_log_internal_check_ok_goo; \ - absl_log_internal_check_ok_goo.first = \ - ::absl::log_internal::AsStatus(val), \ - absl_log_internal_check_ok_goo.second = \ - ABSL_PREDICT_TRUE(absl_log_internal_check_ok_goo.first->ok()) \ - ? nullptr \ - : ::absl::status_internal::MakeCheckFailString( \ - absl_log_internal_check_ok_goo.first, \ - ABSL_LOG_INTERNAL_STRIP_STRING_LITERAL(val_text \ - " is OK")), \ - !ABSL_PREDICT_TRUE(absl_log_internal_check_ok_goo.first->ok());) \ - ABSL_LOG_INTERNAL_QCHECK(*absl_log_internal_check_ok_goo.second) \ +#define ABSL_LOG_INTERNAL_QCHECK_OK(val, val_text) \ + for (::std::pair<const ::absl::Status*, ::std::string*> \ + absl_log_internal_qcheck_ok_goo; \ + absl_log_internal_qcheck_ok_goo.first = \ + ::absl::log_internal::AsStatus(val), \ + absl_log_internal_qcheck_ok_goo.second = \ + ABSL_PREDICT_TRUE(absl_log_internal_qcheck_ok_goo.first->ok()) \ + ? nullptr \ + : ::absl::status_internal::MakeCheckFailString( \ + absl_log_internal_qcheck_ok_goo.first, \ + ABSL_LOG_INTERNAL_STRIP_STRING_LITERAL(val_text \ + " is OK")), \ + !ABSL_PREDICT_TRUE(absl_log_internal_qcheck_ok_goo.first->ok());) \ + ABSL_LOG_INTERNAL_CONDITION_QFATAL(STATELESS, true) \ + ABSL_LOG_INTERNAL_QCHECK(*absl_log_internal_qcheck_ok_goo.second) \ .InternalStream() namespace absl { @@ -152,8 +162,8 @@ template <typename T> class StatusOr; namespace status_internal { -std::string* MakeCheckFailString(const absl::Status* status, - const char* prefix); +ABSL_ATTRIBUTE_PURE_FUNCTION std::string* MakeCheckFailString( + const absl::Status* status, const char* prefix); } // namespace status_internal namespace log_internal { @@ -314,6 +324,20 @@ ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN(const unsigned char*); ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN(const void*); #undef ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN +// `ABSL_LOG_INTERNAL_CHECK_OP_IMPL_RESULT` skips formatting the Check_OP result +// string iff `ABSL_MIN_LOG_LEVEL` exceeds `kFatal`, instead returning an empty +// string. +#ifdef ABSL_MIN_LOG_LEVEL +#define ABSL_LOG_INTERNAL_CHECK_OP_IMPL_RESULT(U1, U2, v1, v2, exprtext) \ + ((::absl::LogSeverity::kFatal >= \ + static_cast<::absl::LogSeverity>(ABSL_MIN_LOG_LEVEL)) \ + ? MakeCheckOpString<U1, U2>(v1, v2, exprtext) \ + : new std::string()) +#else +#define ABSL_LOG_INTERNAL_CHECK_OP_IMPL_RESULT(U1, U2, v1, v2, exprtext) \ + MakeCheckOpString<U1, U2>(v1, v2, exprtext) +#endif + // Helper functions for `ABSL_LOG_INTERNAL_CHECK_OP` macro family. The // `(int, int)` override works around the issue that the compiler will not // instantiate the template version of the function on values of unnamed enum @@ -326,7 +350,8 @@ ABSL_LOG_INTERNAL_DEFINE_MAKE_CHECK_OP_STRING_EXTERN(const void*); using U2 = CheckOpStreamType<T2>; \ return ABSL_PREDICT_TRUE(v1 op v2) \ ? nullptr \ - : MakeCheckOpString<U1, U2>(v1, v2, exprtext); \ + : ABSL_LOG_INTERNAL_CHECK_OP_IMPL_RESULT(U1, U2, v1, v2, \ + exprtext); \ } \ inline constexpr ::std::string* name##Impl(int v1, int v2, \ const char* exprtext) { \ @@ -339,6 +364,7 @@ ABSL_LOG_INTERNAL_CHECK_OP_IMPL(Check_LE, <=) ABSL_LOG_INTERNAL_CHECK_OP_IMPL(Check_LT, <) ABSL_LOG_INTERNAL_CHECK_OP_IMPL(Check_GE, >=) ABSL_LOG_INTERNAL_CHECK_OP_IMPL(Check_GT, >) +#undef ABSL_LOG_INTERNAL_CHECK_OP_IMPL_RESULT #undef ABSL_LOG_INTERNAL_CHECK_OP_IMPL std::string* CheckstrcmptrueImpl(const char* s1, const char* s2, diff --git a/contrib/restricted/abseil-cpp/absl/log/internal/conditions.h b/contrib/restricted/abseil-cpp/absl/log/internal/conditions.h index f576d6500b..645f3c23ea 100644 --- a/contrib/restricted/abseil-cpp/absl/log/internal/conditions.h +++ b/contrib/restricted/abseil-cpp/absl/log/internal/conditions.h @@ -57,7 +57,7 @@ // (dangling else, missing switch case) and preserving noreturn semantics (e.g. // on `LOG(FATAL)`) without requiring braces. // -// The `switch` ensures that this expansion is the begnning of a statement (as +// The `switch` ensures that this expansion is the beginning of a statement (as // opposed to an expression) and prevents shenanigans like // `AFunction(LOG(INFO))` and `decltype(LOG(INFO))`. The apparently-redundant // `default` case makes the condition more amenable to Clang dataflow analysis. @@ -68,7 +68,7 @@ !(condition) ? (void)0 : ::absl::log_internal::Voidify()&& // `ABSL_LOG_INTERNAL_STATEFUL_CONDITION` applies a condition like -// `ABSL_LOG_INTERNAL_CONDITION` but adds to that a series of variable +// `ABSL_LOG_INTERNAL_STATELESS_CONDITION` but adds to that a series of variable // declarations, including a local static object which stores the state needed // to implement the stateful macros like `LOG_EVERY_N`. // @@ -137,21 +137,30 @@ ? true \ : (::absl::log_internal::ExitQuietly(), false)) \ : false)) - -#define ABSL_LOG_INTERNAL_CONDITION_LEVEL(severity) \ - for (int log_internal_severity_loop = 1; log_internal_severity_loop; \ - log_internal_severity_loop = 0) \ - for (const absl::LogSeverity log_internal_severity = \ - ::absl::NormalizeLogSeverity(severity); \ - log_internal_severity_loop; log_internal_severity_loop = 0) \ +#define ABSL_LOG_INTERNAL_CONDITION_DFATAL(type, condition) \ + ABSL_LOG_INTERNAL_##type##_CONDITION( \ + (ABSL_ASSUME(absl::kLogDebugFatal == absl::LogSeverity::kError || \ + absl::kLogDebugFatal == absl::LogSeverity::kFatal), \ + (condition) && \ + (::absl::kLogDebugFatal >= \ + static_cast<::absl::LogSeverity>(ABSL_MIN_LOG_LEVEL) || \ + (::absl::kLogDebugFatal == ::absl::LogSeverity::kFatal && \ + (::absl::log_internal::AbortQuietly(), false))))) + +#define ABSL_LOG_INTERNAL_CONDITION_LEVEL(severity) \ + for (int absl_log_internal_severity_loop = 1; \ + absl_log_internal_severity_loop; absl_log_internal_severity_loop = 0) \ + for (const absl::LogSeverity absl_log_internal_severity = \ + ::absl::NormalizeLogSeverity(severity); \ + absl_log_internal_severity_loop; absl_log_internal_severity_loop = 0) \ ABSL_LOG_INTERNAL_CONDITION_LEVEL_IMPL -#define ABSL_LOG_INTERNAL_CONDITION_LEVEL_IMPL(type, condition) \ - ABSL_LOG_INTERNAL_##type##_CONDITION( \ - (condition) && \ - (log_internal_severity >= \ - static_cast<::absl::LogSeverity>(ABSL_MIN_LOG_LEVEL) || \ - (log_internal_severity == ::absl::LogSeverity::kFatal && \ - (::absl::log_internal::AbortQuietly(), false)))) +#define ABSL_LOG_INTERNAL_CONDITION_LEVEL_IMPL(type, condition) \ + ABSL_LOG_INTERNAL_##type##_CONDITION(( \ + (condition) && \ + (absl_log_internal_severity >= \ + static_cast<::absl::LogSeverity>(ABSL_MIN_LOG_LEVEL) || \ + (absl_log_internal_severity == ::absl::LogSeverity::kFatal && \ + (::absl::log_internal::AbortQuietly(), false))))) #else // ndef ABSL_MIN_LOG_LEVEL #define ABSL_LOG_INTERNAL_CONDITION_INFO(type, condition) \ ABSL_LOG_INTERNAL_##type##_CONDITION(condition) @@ -163,12 +172,14 @@ ABSL_LOG_INTERNAL_##type##_CONDITION(condition) #define ABSL_LOG_INTERNAL_CONDITION_QFATAL(type, condition) \ ABSL_LOG_INTERNAL_##type##_CONDITION(condition) -#define ABSL_LOG_INTERNAL_CONDITION_LEVEL(severity) \ - for (int log_internal_severity_loop = 1; log_internal_severity_loop; \ - log_internal_severity_loop = 0) \ - for (const absl::LogSeverity log_internal_severity = \ - ::absl::NormalizeLogSeverity(severity); \ - log_internal_severity_loop; log_internal_severity_loop = 0) \ +#define ABSL_LOG_INTERNAL_CONDITION_DFATAL(type, condition) \ + ABSL_LOG_INTERNAL_##type##_CONDITION(condition) +#define ABSL_LOG_INTERNAL_CONDITION_LEVEL(severity) \ + for (int absl_log_internal_severity_loop = 1; \ + absl_log_internal_severity_loop; absl_log_internal_severity_loop = 0) \ + for (const absl::LogSeverity absl_log_internal_severity = \ + ::absl::NormalizeLogSeverity(severity); \ + absl_log_internal_severity_loop; absl_log_internal_severity_loop = 0) \ ABSL_LOG_INTERNAL_CONDITION_LEVEL_IMPL #define ABSL_LOG_INTERNAL_CONDITION_LEVEL_IMPL(type, condition) \ ABSL_LOG_INTERNAL_##type##_CONDITION(condition) diff --git a/contrib/restricted/abseil-cpp/absl/log/internal/flags.h b/contrib/restricted/abseil-cpp/absl/log/internal/flags.h index 0c5e81edee..c4539785ad 100644 --- a/contrib/restricted/abseil-cpp/absl/log/internal/flags.h +++ b/contrib/restricted/abseil-cpp/absl/log/internal/flags.h @@ -33,7 +33,7 @@ // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! // Log messages at this severity or above are sent to stderr in *addition* to -// logfiles. Defaults to `ERROR`. See log_severity.h for numeric values of +// `LogSink`s. Defaults to `ERROR`. See log_severity.h for numeric values of // severity levels. ABSL_DECLARE_FLAG(int, stderrthreshold); @@ -50,4 +50,10 @@ ABSL_DECLARE_FLAG(std::string, log_backtrace_at); // each message logged. Defaults to true. ABSL_DECLARE_FLAG(bool, log_prefix); +// Global log verbosity level. Default is 0. +ABSL_DECLARE_FLAG(int, v); + +// Per-module log verbosity level. By default is empty and is unused. +ABSL_DECLARE_FLAG(std::string, vmodule); + #endif // ABSL_LOG_INTERNAL_FLAGS_H_ diff --git a/contrib/restricted/abseil-cpp/absl/log/internal/fnmatch.cc b/contrib/restricted/abseil-cpp/absl/log/internal/fnmatch.cc new file mode 100644 index 0000000000..26e1e57fe4 --- /dev/null +++ b/contrib/restricted/abseil-cpp/absl/log/internal/fnmatch.cc @@ -0,0 +1,73 @@ +// Copyright 2023 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/log/internal/fnmatch.h" + +#include <cstddef> + +#include "absl/base/config.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace log_internal { +bool FNMatch(absl::string_view pattern, absl::string_view str) { + bool in_wildcard_match = false; + while (true) { + if (pattern.empty()) { + // `pattern` is exhausted; succeed if all of `str` was consumed matching + // it. + return in_wildcard_match || str.empty(); + } + if (str.empty()) { + // `str` is exhausted; succeed if `pattern` is empty or all '*'s. + return pattern.find_first_not_of('*') == pattern.npos; + } + switch (pattern.front()) { + case '*': + pattern.remove_prefix(1); + in_wildcard_match = true; + break; + case '?': + pattern.remove_prefix(1); + str.remove_prefix(1); + break; + default: + if (in_wildcard_match) { + absl::string_view fixed_portion = pattern; + const size_t end = fixed_portion.find_first_of("*?"); + if (end != fixed_portion.npos) { + fixed_portion = fixed_portion.substr(0, end); + } + const size_t match = str.find(fixed_portion); + if (match == str.npos) { + return false; + } + pattern.remove_prefix(fixed_portion.size()); + str.remove_prefix(match + fixed_portion.size()); + in_wildcard_match = false; + } else { + if (pattern.front() != str.front()) { + return false; + } + pattern.remove_prefix(1); + str.remove_prefix(1); + } + break; + } + } +} +} // namespace log_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/contrib/restricted/abseil-cpp/absl/flags/flag.cc b/contrib/restricted/abseil-cpp/absl/log/internal/fnmatch.h index 531df1287a..4ea147cc6f 100644 --- a/contrib/restricted/abseil-cpp/absl/flags/flag.cc +++ b/contrib/restricted/abseil-cpp/absl/log/internal/fnmatch.h @@ -1,11 +1,10 @@ -// -// Copyright 2019 The Abseil Authors. +// Copyright 2023 The Abseil Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// https://www.apache.org/licenses/LICENSE-2.0 +// https://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -13,26 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "absl/flags/flag.h" +#ifndef ABSL_LOG_INTERNAL_FNMATCH_H_ +#define ABSL_LOG_INTERNAL_FNMATCH_H_ #include "absl/base/config.h" +#include "absl/strings/string_view.h" namespace absl { ABSL_NAMESPACE_BEGIN - -// This global mutex protects on-demand construction of flag objects in MSVC -// builds. -#if defined(_MSC_VER) && !defined(__clang__) - -namespace flags_internal { - -ABSL_CONST_INIT static absl::Mutex construction_guard(absl::kConstInit); - -absl::Mutex* GetGlobalConstructionGuard() { return &construction_guard; } - -} // namespace flags_internal - -#endif - +namespace log_internal { +// Like POSIX `fnmatch`, but: +// * accepts `string_view` +// * does not allocate any dynamic memory +// * only supports * and ? wildcards and not bracket expressions [...] +// * wildcards may match / +// * no backslash-escaping +bool FNMatch(absl::string_view pattern, absl::string_view str); +} // namespace log_internal ABSL_NAMESPACE_END } // namespace absl + +#endif // ABSL_LOG_INTERNAL_FNMATCH_H_ diff --git a/contrib/restricted/abseil-cpp/absl/log/internal/log_impl.h b/contrib/restricted/abseil-cpp/absl/log/internal/log_impl.h index 9326780dc6..99de6dbb24 100644 --- a/contrib/restricted/abseil-cpp/absl/log/internal/log_impl.h +++ b/contrib/restricted/abseil-cpp/absl/log/internal/log_impl.h @@ -15,6 +15,7 @@ #ifndef ABSL_LOG_INTERNAL_LOG_IMPL_H_ #define ABSL_LOG_INTERNAL_LOG_IMPL_H_ +#include "absl/log/absl_vlog_is_on.h" #include "absl/log/internal/conditions.h" #include "absl/log/internal/log_message.h" #include "absl/log/internal/strip.h" @@ -41,6 +42,35 @@ ABSL_LOGGING_INTERNAL_LOG##severity.InternalStream() #endif +// The `switch` ensures that this expansion is the begnning of a statement (as +// opposed to an expression). The use of both `case 0` and `default` is to +// suppress a compiler warning. +#define ABSL_LOG_INTERNAL_VLOG_IMPL(verbose_level) \ + switch (const int absl_logging_internal_verbose_level = (verbose_level)) \ + case 0: \ + default: \ + ABSL_LOG_INTERNAL_LOG_IF_IMPL( \ + _INFO, ABSL_VLOG_IS_ON(absl_logging_internal_verbose_level)) \ + .WithVerbosity(absl_logging_internal_verbose_level) + +#ifndef NDEBUG +#define ABSL_LOG_INTERNAL_DVLOG_IMPL(verbose_level) \ + switch (const int absl_logging_internal_verbose_level = (verbose_level)) \ + case 0: \ + default: \ + ABSL_LOG_INTERNAL_LOG_IF_IMPL( \ + _INFO, ABSL_VLOG_IS_ON(absl_logging_internal_verbose_level)) \ + .WithVerbosity(absl_logging_internal_verbose_level) +#else +#define ABSL_LOG_INTERNAL_DVLOG_IMPL(verbose_level) \ + switch (const int absl_logging_internal_verbose_level = (verbose_level)) \ + case 0: \ + default: \ + ABSL_LOG_INTERNAL_LOG_IF_IMPL( \ + _INFO, false && ABSL_VLOG_IS_ON(absl_logging_internal_verbose_level)) \ + .WithVerbosity(absl_logging_internal_verbose_level) +#endif + #define ABSL_LOG_INTERNAL_LOG_IF_IMPL(severity, condition) \ ABSL_LOG_INTERNAL_CONDITION##severity(STATELESS, condition) \ ABSL_LOGGING_INTERNAL_LOG##severity.InternalStream() @@ -134,6 +164,42 @@ (EveryNSec, n_seconds) ABSL_LOGGING_INTERNAL_LOG##severity.InternalStream() #endif // def NDEBUG +#define ABSL_LOG_INTERNAL_VLOG_EVERY_N_IMPL(verbose_level, n) \ + switch (const int absl_logging_internal_verbose_level = (verbose_level)) \ + case 0: \ + default: \ + ABSL_LOG_INTERNAL_CONDITION_INFO( \ + STATEFUL, ABSL_VLOG_IS_ON(absl_logging_internal_verbose_level)) \ + (EveryN, n) ABSL_LOGGING_INTERNAL_LOG_INFO.InternalStream().WithVerbosity( \ + absl_logging_internal_verbose_level) + +#define ABSL_LOG_INTERNAL_VLOG_FIRST_N_IMPL(verbose_level, n) \ + switch (const int absl_logging_internal_verbose_level = (verbose_level)) \ + case 0: \ + default: \ + ABSL_LOG_INTERNAL_CONDITION_INFO( \ + STATEFUL, ABSL_VLOG_IS_ON(absl_logging_internal_verbose_level)) \ + (FirstN, n) ABSL_LOGGING_INTERNAL_LOG_INFO.InternalStream().WithVerbosity( \ + absl_logging_internal_verbose_level) + +#define ABSL_LOG_INTERNAL_VLOG_EVERY_POW_2_IMPL(verbose_level) \ + switch (const int absl_logging_internal_verbose_level = (verbose_level)) \ + case 0: \ + default: \ + ABSL_LOG_INTERNAL_CONDITION_INFO( \ + STATEFUL, ABSL_VLOG_IS_ON(absl_logging_internal_verbose_level)) \ + (EveryPow2) ABSL_LOGGING_INTERNAL_LOG_INFO.InternalStream().WithVerbosity( \ + absl_logging_internal_verbose_level) + +#define ABSL_LOG_INTERNAL_VLOG_EVERY_N_SEC_IMPL(verbose_level, n_seconds) \ + switch (const int absl_logging_internal_verbose_level = (verbose_level)) \ + case 0: \ + default: \ + ABSL_LOG_INTERNAL_CONDITION_INFO( \ + STATEFUL, ABSL_VLOG_IS_ON(absl_logging_internal_verbose_level)) \ + (EveryNSec, n_seconds) ABSL_LOGGING_INTERNAL_LOG_INFO.InternalStream() \ + .WithVerbosity(absl_logging_internal_verbose_level) + #define ABSL_LOG_INTERNAL_LOG_IF_EVERY_N_IMPL(severity, condition, n) \ ABSL_LOG_INTERNAL_CONDITION##severity(STATEFUL, condition)(EveryN, n) \ ABSL_LOGGING_INTERNAL_LOG##severity.InternalStream() diff --git a/contrib/restricted/abseil-cpp/absl/log/internal/log_message.h b/contrib/restricted/abseil-cpp/absl/log/internal/log_message.h index 46937728cb..4ecb8a1498 100644 --- a/contrib/restricted/abseil-cpp/absl/log/internal/log_message.h +++ b/contrib/restricted/abseil-cpp/absl/log/internal/log_message.h @@ -40,7 +40,7 @@ #include "absl/log/internal/nullguard.h" #include "absl/log/log_entry.h" #include "absl/log/log_sink.h" -#include "absl/strings/internal/has_absl_stringify.h" +#include "absl/strings/has_absl_stringify.h" #include "absl/strings/string_view.h" #include "absl/time/time.h" @@ -170,15 +170,15 @@ class LogMessage { // Types that support `AbslStringify()` are serialized that way. template <typename T, - typename std::enable_if< - strings_internal::HasAbslStringify<T>::value, int>::type = 0> + typename std::enable_if<absl::HasAbslStringify<T>::value, + int>::type = 0> LogMessage& operator<<(const T& v) ABSL_ATTRIBUTE_NOINLINE; // Types that don't support `AbslStringify()` but do support streaming into a // `std::ostream&` are serialized that way. template <typename T, - typename std::enable_if< - !strings_internal::HasAbslStringify<T>::value, int>::type = 0> + typename std::enable_if<!absl::HasAbslStringify<T>::value, + int>::type = 0> LogMessage& operator<<(const T& v) ABSL_ATTRIBUTE_NOINLINE; // Note: We explicitly do not support `operator<<` for non-const references @@ -283,8 +283,7 @@ class StringifySink final { // Note: the following is declared `ABSL_ATTRIBUTE_NOINLINE` template <typename T, - typename std::enable_if<strings_internal::HasAbslStringify<T>::value, - int>::type> + typename std::enable_if<absl::HasAbslStringify<T>::value, int>::type> LogMessage& LogMessage::operator<<(const T& v) { StringifySink sink(*this); // Replace with public API. @@ -294,8 +293,7 @@ LogMessage& LogMessage::operator<<(const T& v) { // Note: the following is declared `ABSL_ATTRIBUTE_NOINLINE` template <typename T, - typename std::enable_if<!strings_internal::HasAbslStringify<T>::value, - int>::type> + typename std::enable_if<!absl::HasAbslStringify<T>::value, int>::type> LogMessage& LogMessage::operator<<(const T& v) { OstreamView view(*data_); view.stream() << log_internal::NullGuard<T>().Guard(v); diff --git a/contrib/restricted/abseil-cpp/absl/log/internal/log_sink_set.cc b/contrib/restricted/abseil-cpp/absl/log/internal/log_sink_set.cc index b7cbe36455..3d5c69952c 100644 --- a/contrib/restricted/abseil-cpp/absl/log/internal/log_sink_set.cc +++ b/contrib/restricted/abseil-cpp/absl/log/internal/log_sink_set.cc @@ -35,6 +35,7 @@ #include "absl/base/config.h" #include "absl/base/internal/raw_logging.h" #include "absl/base/log_severity.h" +#include "absl/base/no_destructor.h" #include "absl/base/thread_annotations.h" #include "absl/cleanup/cleanup.h" #include "absl/log/globals.h" @@ -168,17 +169,16 @@ class GlobalLogSinkSet final { #if defined(__myriad2__) || defined(__Fuchsia__) // myriad2 and Fuchsia do not log to stderr by default. #else - static StderrLogSink* stderr_log_sink = new StderrLogSink; - AddLogSink(stderr_log_sink); + static absl::NoDestructor<StderrLogSink> stderr_log_sink; + AddLogSink(stderr_log_sink.get()); #endif #ifdef __ANDROID__ - static AndroidLogSink* android_log_sink = new AndroidLogSink; - AddLogSink(android_log_sink); + static absl::NoDestructor<AndroidLogSink> android_log_sink; + AddLogSink(android_log_sink.get()); #endif #if defined(_WIN32) - static WindowsDebuggerLogSink* debugger_log_sink = - new WindowsDebuggerLogSink; - AddLogSink(debugger_log_sink); + static absl::NoDestructor<WindowsDebuggerLogSink> debugger_log_sink; + AddLogSink(debugger_log_sink.get()); #endif // !defined(_WIN32) } @@ -268,7 +268,7 @@ class GlobalLogSinkSet final { // Returns reference to the global LogSinks set. GlobalLogSinkSet& GlobalSinks() { - static GlobalLogSinkSet* global_sinks = new GlobalLogSinkSet; + static absl::NoDestructor<GlobalLogSinkSet> global_sinks; return *global_sinks; } diff --git a/contrib/restricted/abseil-cpp/absl/log/internal/strip.h b/contrib/restricted/abseil-cpp/absl/log/internal/strip.h index adc86ffdfa..f8d2786939 100644 --- a/contrib/restricted/abseil-cpp/absl/log/internal/strip.h +++ b/contrib/restricted/abseil-cpp/absl/log/internal/strip.h @@ -37,7 +37,7 @@ #define ABSL_LOGGING_INTERNAL_LOG_DFATAL \ ::absl::log_internal::NullStreamMaybeFatal(::absl::kLogDebugFatal) #define ABSL_LOGGING_INTERNAL_LOG_LEVEL(severity) \ - ::absl::log_internal::NullStreamMaybeFatal(log_internal_severity) + ::absl::log_internal::NullStreamMaybeFatal(absl_log_internal_severity) #define ABSL_LOG_INTERNAL_CHECK(failure_message) ABSL_LOGGING_INTERNAL_LOG_FATAL #define ABSL_LOG_INTERNAL_QCHECK(failure_message) \ ABSL_LOGGING_INTERNAL_LOG_QFATAL @@ -57,8 +57,9 @@ ::absl::log_internal::LogMessageQuietlyFatal(__FILE__, __LINE__) #define ABSL_LOGGING_INTERNAL_LOG_DFATAL \ ::absl::log_internal::LogMessage(__FILE__, __LINE__, ::absl::kLogDebugFatal) -#define ABSL_LOGGING_INTERNAL_LOG_LEVEL(severity) \ - ::absl::log_internal::LogMessage(__FILE__, __LINE__, log_internal_severity) +#define ABSL_LOGGING_INTERNAL_LOG_LEVEL(severity) \ + ::absl::log_internal::LogMessage(__FILE__, __LINE__, \ + absl_log_internal_severity) // These special cases dispatch to special-case constructors that allow us to // avoid an extra function call and shrink non-LTO binaries by a percent or so. #define ABSL_LOG_INTERNAL_CHECK(failure_message) \ diff --git a/contrib/restricted/abseil-cpp/absl/log/internal/vlog_config.cc b/contrib/restricted/abseil-cpp/absl/log/internal/vlog_config.cc new file mode 100644 index 0000000000..b578850041 --- /dev/null +++ b/contrib/restricted/abseil-cpp/absl/log/internal/vlog_config.cc @@ -0,0 +1,340 @@ +// Copyright 2022 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/log/internal/vlog_config.h" + +#include <stddef.h> + +#include <algorithm> +#include <atomic> +#include <functional> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/const_init.h" +#include "absl/base/internal/spinlock.h" +#include "absl/base/no_destructor.h" +#include "absl/base/optimization.h" +#include "absl/base/thread_annotations.h" +#include "absl/log/internal/fnmatch.h" +#include "absl/memory/memory.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/strings/strip.h" +#include "absl/synchronization/mutex.h" +#include "absl/types/optional.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace log_internal { + +namespace { +bool ModuleIsPath(absl::string_view module_pattern) { +#ifdef _WIN32 + return module_pattern.find_first_of("/\\") != module_pattern.npos; +#else + return module_pattern.find('/') != module_pattern.npos; +#endif +} +} // namespace + +bool VLogSite::SlowIsEnabled(int stale_v, int level) { + if (ABSL_PREDICT_TRUE(stale_v != kUninitialized)) { + // Because of the prerequisites to this function, we know that stale_v is + // either uninitialized or >= level. If it's not uninitialized, that means + // it must be >= level, thus we should log. + return true; + } + stale_v = log_internal::RegisterAndInitialize(this); + return ABSL_PREDICT_FALSE(stale_v >= level); +} + +bool VLogSite::SlowIsEnabled0(int stale_v) { return SlowIsEnabled(stale_v, 0); } +bool VLogSite::SlowIsEnabled1(int stale_v) { return SlowIsEnabled(stale_v, 1); } +bool VLogSite::SlowIsEnabled2(int stale_v) { return SlowIsEnabled(stale_v, 2); } +bool VLogSite::SlowIsEnabled3(int stale_v) { return SlowIsEnabled(stale_v, 3); } +bool VLogSite::SlowIsEnabled4(int stale_v) { return SlowIsEnabled(stale_v, 4); } +bool VLogSite::SlowIsEnabled5(int stale_v) { return SlowIsEnabled(stale_v, 5); } + +namespace { +struct VModuleInfo final { + std::string module_pattern; + bool module_is_path; // i.e. it contains a path separator. + int vlog_level; + + // Allocates memory. + VModuleInfo(absl::string_view module_pattern_arg, bool module_is_path_arg, + int vlog_level_arg) + : module_pattern(std::string(module_pattern_arg)), + module_is_path(module_is_path_arg), + vlog_level(vlog_level_arg) {} +}; + +// `mutex` guards all of the data structures that aren't lock-free. +// To avoid problems with the heap checker which calls into `VLOG`, `mutex` must +// be a `SpinLock` that prevents fiber scheduling instead of a `Mutex`. +ABSL_CONST_INIT absl::base_internal::SpinLock mutex( + absl::kConstInit, absl::base_internal::SCHEDULE_KERNEL_ONLY); + +// `GetUpdateSitesMutex()` serializes updates to all of the sites (i.e. those in +// `site_list_head`) themselves. +absl::Mutex* GetUpdateSitesMutex() { + // Chromium requires no global destructors, so we can't use the + // absl::kConstInit idiom since absl::Mutex as a non-trivial destructor. + static absl::NoDestructor<absl::Mutex> update_sites_mutex ABSL_ACQUIRED_AFTER( + mutex); + return update_sites_mutex.get(); +} + +ABSL_CONST_INIT int global_v ABSL_GUARDED_BY(mutex) = 0; +// `site_list_head` is the head of a singly-linked list. Traversal, insertion, +// and reads are atomic, so no locks are required, but updates to existing +// elements are guarded by `GetUpdateSitesMutex()`. +ABSL_CONST_INIT std::atomic<VLogSite*> site_list_head{nullptr}; +ABSL_CONST_INIT std::vector<VModuleInfo>* vmodule_info ABSL_GUARDED_BY(mutex) + ABSL_PT_GUARDED_BY(mutex){nullptr}; + +// Only used for lisp. +ABSL_CONST_INIT std::vector<std::function<void()>>* update_callbacks + ABSL_GUARDED_BY(GetUpdateSitesMutex()) + ABSL_PT_GUARDED_BY(GetUpdateSitesMutex()){nullptr}; + +// Allocates memory. +std::vector<VModuleInfo>& get_vmodule_info() + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex) { + if (!vmodule_info) vmodule_info = new std::vector<VModuleInfo>; + return *vmodule_info; +} + +// Does not allocate or take locks. +int VLogLevel(absl::string_view file, const std::vector<VModuleInfo>* infos, + int current_global_v) { + // `infos` is null during a call to `VLOG` prior to setting `vmodule` (e.g. by + // parsing flags). We can't allocate in `VLOG`, so we treat null as empty + // here and press on. + if (!infos || infos->empty()) return current_global_v; + // Get basename for file + absl::string_view basename = file; + { + const size_t sep = basename.rfind('/'); + if (sep != basename.npos) { + basename.remove_prefix(sep + 1); +#ifdef _WIN32 + } else { + const size_t sep = basename.rfind('\\'); + if (sep != basename.npos) basename.remove_prefix(sep + 1); +#endif + } + } + + absl::string_view stem = file, stem_basename = basename; + { + const size_t sep = stem_basename.find('.'); + if (sep != stem_basename.npos) { + stem.remove_suffix(stem_basename.size() - sep); + stem_basename.remove_suffix(stem_basename.size() - sep); + } + if (absl::ConsumeSuffix(&stem_basename, "-inl")) { + stem.remove_suffix(absl::string_view("-inl").size()); + } + } + for (const auto& info : *infos) { + if (info.module_is_path) { + // If there are any slashes in the pattern, try to match the full + // name. + if (FNMatch(info.module_pattern, stem)) { + return info.vlog_level == kUseFlag ? current_global_v : info.vlog_level; + } + } else if (FNMatch(info.module_pattern, stem_basename)) { + return info.vlog_level == kUseFlag ? current_global_v : info.vlog_level; + } + } + + return current_global_v; +} + +// Allocates memory. +int AppendVModuleLocked(absl::string_view module_pattern, int log_level) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex) { + for (const auto& info : get_vmodule_info()) { + if (FNMatch(info.module_pattern, module_pattern)) { + // This is a memory optimization to avoid storing patterns that will never + // match due to exit early semantics. Primarily optimized for our own unit + // tests. + return info.vlog_level; + } + } + bool module_is_path = ModuleIsPath(module_pattern); + get_vmodule_info().emplace_back(std::string(module_pattern), module_is_path, + log_level); + return global_v; +} + +// Allocates memory. +int PrependVModuleLocked(absl::string_view module_pattern, int log_level) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex) { + absl::optional<int> old_log_level; + for (const auto& info : get_vmodule_info()) { + if (FNMatch(info.module_pattern, module_pattern)) { + old_log_level = info.vlog_level; + break; + } + } + bool module_is_path = ModuleIsPath(module_pattern); + auto iter = get_vmodule_info().emplace(get_vmodule_info().cbegin(), + std::string(module_pattern), + module_is_path, log_level); + + // This is a memory optimization to avoid storing patterns that will never + // match due to exit early semantics. Primarily optimized for our own unit + // tests. + get_vmodule_info().erase( + std::remove_if(++iter, get_vmodule_info().end(), + [module_pattern](const VModuleInfo& info) { + return FNMatch(info.module_pattern, module_pattern); + }), + get_vmodule_info().cend()); + return old_log_level.value_or(global_v); +} +} // namespace + +int VLogLevel(absl::string_view file) ABSL_LOCKS_EXCLUDED(mutex) { + absl::base_internal::SpinLockHolder l(&mutex); + return VLogLevel(file, vmodule_info, global_v); +} + +int RegisterAndInitialize(VLogSite* v) ABSL_LOCKS_EXCLUDED(mutex) { + // std::memory_order_seq_cst is overkill in this function, but given that this + // path is intended to be slow, it's not worth the brain power to relax that. + VLogSite* h = site_list_head.load(std::memory_order_seq_cst); + + VLogSite* old = nullptr; + if (v->next_.compare_exchange_strong(old, h, std::memory_order_seq_cst, + std::memory_order_seq_cst)) { + // Multiple threads may attempt to register this site concurrently. + // By successfully setting `v->next` this thread commits to being *the* + // thread that installs `v` in the list. + while (!site_list_head.compare_exchange_weak( + h, v, std::memory_order_seq_cst, std::memory_order_seq_cst)) { + v->next_.store(h, std::memory_order_seq_cst); + } + } + + int old_v = VLogSite::kUninitialized; + int new_v = VLogLevel(v->file_); + // No loop, if someone else set this, we should respect their evaluation of + // `VLogLevel`. This may mean we return a stale `v`, but `v` itself will + // always arrive at the freshest value. Otherwise, we could be writing a + // stale value and clobbering the fresher one. + if (v->v_.compare_exchange_strong(old_v, new_v, std::memory_order_seq_cst, + std::memory_order_seq_cst)) { + return new_v; + } + return old_v; +} + +void UpdateVLogSites() ABSL_UNLOCK_FUNCTION(mutex) + ABSL_LOCKS_EXCLUDED(GetUpdateSitesMutex()) { + std::vector<VModuleInfo> infos = get_vmodule_info(); + int current_global_v = global_v; + // We need to grab `GetUpdateSitesMutex()` before we release `mutex` to ensure + // that updates are not interleaved (resulting in an inconsistent final state) + // and to ensure that the final state in the sites matches the final state of + // `vmodule_info`. We unlock `mutex` to ensure that uninitialized sites don't + // have to wait on all updates in order to acquire `mutex` and initialize + // themselves. + absl::MutexLock ul(GetUpdateSitesMutex()); + mutex.Unlock(); + VLogSite* n = site_list_head.load(std::memory_order_seq_cst); + // Because sites are added to the list in the order they are executed, there + // tend to be clusters of entries with the same file. + const char* last_file = nullptr; + int last_file_level = 0; + while (n != nullptr) { + if (n->file_ != last_file) { + last_file = n->file_; + last_file_level = VLogLevel(n->file_, &infos, current_global_v); + } + n->v_.store(last_file_level, std::memory_order_seq_cst); + n = n->next_.load(std::memory_order_seq_cst); + } + if (update_callbacks) { + for (auto& cb : *update_callbacks) { + cb(); + } + } +} + +void UpdateVModule(absl::string_view vmodule) + ABSL_LOCKS_EXCLUDED(mutex, GetUpdateSitesMutex()) { + std::vector<std::pair<absl::string_view, int>> glob_levels; + for (absl::string_view glob_level : absl::StrSplit(vmodule, ',')) { + const size_t eq = glob_level.rfind('='); + if (eq == glob_level.npos) continue; + const absl::string_view glob = glob_level.substr(0, eq); + int level; + if (!absl::SimpleAtoi(glob_level.substr(eq + 1), &level)) continue; + glob_levels.emplace_back(glob, level); + } + mutex.Lock(); // Unlocked by UpdateVLogSites(). + get_vmodule_info().clear(); + for (const auto& it : glob_levels) { + const absl::string_view glob = it.first; + const int level = it.second; + AppendVModuleLocked(glob, level); + } + UpdateVLogSites(); +} + +int UpdateGlobalVLogLevel(int v) + ABSL_LOCKS_EXCLUDED(mutex, GetUpdateSitesMutex()) { + mutex.Lock(); // Unlocked by UpdateVLogSites(). + const int old_global_v = global_v; + if (v == global_v) { + mutex.Unlock(); + return old_global_v; + } + global_v = v; + UpdateVLogSites(); + return old_global_v; +} + +int PrependVModule(absl::string_view module_pattern, int log_level) + ABSL_LOCKS_EXCLUDED(mutex, GetUpdateSitesMutex()) { + mutex.Lock(); // Unlocked by UpdateVLogSites(). + int old_v = PrependVModuleLocked(module_pattern, log_level); + UpdateVLogSites(); + return old_v; +} + +void OnVLogVerbosityUpdate(std::function<void()> cb) + ABSL_LOCKS_EXCLUDED(GetUpdateSitesMutex()) { + absl::MutexLock ul(GetUpdateSitesMutex()); + if (!update_callbacks) + update_callbacks = new std::vector<std::function<void()>>; + update_callbacks->push_back(std::move(cb)); +} + +VLogSite* SetVModuleListHeadForTestOnly(VLogSite* v) { + return site_list_head.exchange(v, std::memory_order_seq_cst); +} + +} // namespace log_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/contrib/restricted/abseil-cpp/absl/log/internal/vlog_config.h b/contrib/restricted/abseil-cpp/absl/log/internal/vlog_config.h new file mode 100644 index 0000000000..b6e322c468 --- /dev/null +++ b/contrib/restricted/abseil-cpp/absl/log/internal/vlog_config.h @@ -0,0 +1,163 @@ +// Copyright 2022 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// vlog_config.h +// ----------------------------------------------------------------------------- +// +// This header file defines `VLogSite`, a public primitive that represents +// a callsite for the `VLOG` family of macros and related libraries. +// It also declares and defines multiple internal utilities used to implement +// `VLOG`, such as `VLogSiteManager`. + +#ifndef ABSL_LOG_INTERNAL_VLOG_CONFIG_H_ +#define ABSL_LOG_INTERNAL_VLOG_CONFIG_H_ + +// IWYU pragma: private, include "absl/log/log.h" + +#include <atomic> +#include <cstdint> +#include <functional> +#include <limits> +#include <type_traits> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/optimization.h" +#include "absl/base/thread_annotations.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace log_internal { + +class SyntheticBinary; +class VLogSite; + +int RegisterAndInitialize(VLogSite* v); +void UpdateVLogSites(); +constexpr int kUseFlag = (std::numeric_limits<int16_t>::min)(); + +// Represents a unique callsite for a `VLOG()` or `VLOG_IS_ON()` call. +// +// Libraries that provide `VLOG`-like functionality should use this to +// efficiently handle --vmodule. +// +// VLogSite objects must not be destroyed until the program exits. Doing so will +// probably yield nasty segfaults in VLogSiteManager::UpdateLogSites(). The +// recommendation is to make all such objects function-local statics. +class VLogSite final { + public: + // `f` must not be destroyed until the program exits. + explicit constexpr VLogSite(const char* f) + : file_(f), v_(kUninitialized), next_(nullptr) {} + VLogSite(const VLogSite&) = delete; + VLogSite& operator=(const VLogSite&) = delete; + + // Inlining the function yields a ~3x performance improvement at the cost of a + // 1.5x code size increase at the call site. + // Takes locks but does not allocate memory. + ABSL_ATTRIBUTE_ALWAYS_INLINE + bool IsEnabled(int level) { + int stale_v = v_.load(std::memory_order_relaxed); + if (ABSL_PREDICT_TRUE(level > stale_v)) { + return false; + } + + // We put everything other than the fast path, i.e. vlogging is initialized + // but not on, behind an out-of-line function to reduce code size. + // "level" is almost always a call-site constant, so we can save a bit + // of code space by special-casing for a few common levels. +#if ABSL_HAVE_BUILTIN(__builtin_constant_p) || defined(__GNUC__) + if (__builtin_constant_p(level)) { + if (level == 0) return SlowIsEnabled0(stale_v); + if (level == 1) return SlowIsEnabled1(stale_v); + if (level == 2) return SlowIsEnabled2(stale_v); + if (level == 3) return SlowIsEnabled3(stale_v); + if (level == 4) return SlowIsEnabled4(stale_v); + if (level == 5) return SlowIsEnabled5(stale_v); + } +#endif + return SlowIsEnabled(stale_v, level); + } + + private: + friend int log_internal::RegisterAndInitialize(VLogSite* v); + friend void log_internal::UpdateVLogSites(); + friend class log_internal::SyntheticBinary; + static constexpr int kUninitialized = (std::numeric_limits<int>::max)(); + + // SlowIsEnabled performs slower checks to determine whether a log site is + // enabled. Because it is expected to be called somewhat rarely + // (comparatively), it is not inlined to save on code size. + // + // Prerequisites to calling SlowIsEnabled: + // 1) stale_v is uninitialized OR + // 2) stale_v is initialized and >= level (meaning we must log). + // Takes locks but does not allocate memory. + ABSL_ATTRIBUTE_NOINLINE + bool SlowIsEnabled(int stale_v, int level); + ABSL_ATTRIBUTE_NOINLINE bool SlowIsEnabled0(int stale_v); + ABSL_ATTRIBUTE_NOINLINE bool SlowIsEnabled1(int stale_v); + ABSL_ATTRIBUTE_NOINLINE bool SlowIsEnabled2(int stale_v); + ABSL_ATTRIBUTE_NOINLINE bool SlowIsEnabled3(int stale_v); + ABSL_ATTRIBUTE_NOINLINE bool SlowIsEnabled4(int stale_v); + ABSL_ATTRIBUTE_NOINLINE bool SlowIsEnabled5(int stale_v); + + // This object is too size-sensitive to use absl::string_view. + const char* const file_; + std::atomic<int> v_; + std::atomic<VLogSite*> next_; +}; +static_assert(std::is_trivially_destructible<VLogSite>::value, + "VLogSite must be trivially destructible"); + +// Returns the current verbose log level of `file`. +// Does not allocate memory. +int VLogLevel(absl::string_view file); + +// Registers a site `v` to get updated as `vmodule` and `v` change. Also +// initializes the site based on their current values, and returns that result. +// Does not allocate memory. +int RegisterAndInitialize(VLogSite* v); + +// Allocates memory. +void UpdateVLogSites(); + +// Completely overwrites the saved value of `vmodule`. +// Allocates memory. +void UpdateVModule(absl::string_view vmodule); + +// Updates the global verbosity level to `v` and returns the prior value. +// Allocates memory. +int UpdateGlobalVLogLevel(int v); + +// Atomically prepends `module_pattern=log_level` to the start of vmodule. +// Returns the prior value for `module_pattern` if there was an exact match and +// `global_v` otherwise. +// Allocates memory. +int PrependVModule(absl::string_view module_pattern, int log_level); + +// Registers `on_update` to be called whenever `v` or `vmodule` change. +// Allocates memory. +void OnVLogVerbosityUpdate(std::function<void()> cb); + +// Does not allocate memory. +VLogSite* SetVModuleListHeadForTestOnly(VLogSite* v); + +} // namespace log_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_LOG_INTERNAL_VLOG_CONFIG_H_ diff --git a/contrib/restricted/abseil-cpp/absl/log/log.h b/contrib/restricted/abseil-cpp/absl/log/log.h index 602b5acf07..b721b087cc 100644 --- a/contrib/restricted/abseil-cpp/absl/log/log.h +++ b/contrib/restricted/abseil-cpp/absl/log/log.h @@ -32,6 +32,8 @@ // * The `QFATAL` pseudo-severity level is equivalent to `FATAL` but triggers // quieter termination messages, e.g. without a full stack trace, and skips // running registered error handlers. +// * The `DFATAL` pseudo-severity level is defined as `FATAL` in debug mode and +// as `ERROR` otherwise. // Some preprocessor shenanigans are used to ensure that e.g. `LOG(INFO)` has // the same meaning even if a local symbol or preprocessor macro named `INFO` is // defined. To specify a severity level using an expression instead of a @@ -51,6 +53,14 @@ // * .NoPrefix() // Omits the prefix from this line. The prefix includes metadata about the // logged data such as source code location and timestamp. +// * .WithVerbosity(int verbose_level) +// Sets the verbosity field of the logged message as if it was logged by +// `VLOG(verbose_level)`. Unlike `VLOG`, this method does not affect +// evaluation of the statement when the specified `verbose_level` has been +// disabled. The only effect is on `LogSink` implementations which make use +// of the `absl::LogSink::verbosity()` value. The value +// `absl::LogEntry::kNoVerbosityLevel` can be specified to mark the message +// not verbose. // * .WithTimestamp(absl::Time timestamp) // Uses the specified timestamp instead of one collected at the time of // execution. @@ -188,6 +198,7 @@ #define ABSL_LOG_LOG_H_ #include "absl/log/internal/log_impl.h" +#include "absl/log/vlog_is_on.h" // IWYU pragma: export // LOG() // @@ -211,11 +222,32 @@ // terminate the program if `NDEBUG` is defined. #define DLOG(severity) ABSL_LOG_INTERNAL_DLOG_IMPL(_##severity) +// `VLOG` uses numeric levels to provide verbose logging that can configured at +// runtime, including at a per-module level. `VLOG` statements are logged at +// `INFO` severity if they are logged at all; the numeric levels are on a +// different scale than the proper severity levels. Positive levels are +// disabled by default. Negative levels should not be used. +// Example: +// +// VLOG(1) << "I print when you run the program with --v=1 or higher"; +// VLOG(2) << "I print when you run the program with --v=2 or higher"; +// +// See vlog_is_on.h for further documentation, including the usage of the +// --vmodule flag to log at different levels in different source files. +#define VLOG(severity) ABSL_LOG_INTERNAL_VLOG_IMPL(severity) + +// `DVLOG` behaves like `VLOG` in debug mode (i.e. `#ifndef NDEBUG`). +// Otherwise, it compiles away and does nothing. +#define DVLOG(severity) ABSL_LOG_INTERNAL_DVLOG_IMPL(severity) + // `LOG_IF` and friends add a second argument which specifies a condition. If // the condition is false, nothing is logged. // Example: // // LOG_IF(INFO, num_cookies > 10) << "Got lots of cookies"; +// +// There is no `VLOG_IF` because the order of evaluation of the arguments is +// ambiguous and the alternate spelling with an `if`-statement is trivial. #define LOG_IF(severity, condition) \ ABSL_LOG_INTERNAL_LOG_IF_IMPL(_##severity, condition) #define PLOG_IF(severity, condition) \ @@ -283,6 +315,15 @@ #define DLOG_EVERY_N_SEC(severity, n_seconds) \ ABSL_LOG_INTERNAL_DLOG_EVERY_N_SEC_IMPL(_##severity, n_seconds) +#define VLOG_EVERY_N(severity, n) \ + ABSL_LOG_INTERNAL_VLOG_EVERY_N_IMPL(severity, n) +#define VLOG_FIRST_N(severity, n) \ + ABSL_LOG_INTERNAL_VLOG_FIRST_N_IMPL(severity, n) +#define VLOG_EVERY_POW_2(severity) \ + ABSL_LOG_INTERNAL_VLOG_EVERY_POW_2_IMPL(severity) +#define VLOG_EVERY_N_SEC(severity, n_seconds) \ + ABSL_LOG_INTERNAL_VLOG_EVERY_N_SEC_IMPL(severity, n_seconds) + // `LOG_IF_EVERY_N` and friends behave as the corresponding `LOG_EVERY_N` // but neither increment a counter nor log a message if condition is false (as // `LOG_IF`). diff --git a/contrib/restricted/abseil-cpp/absl/log/log_entry.cc b/contrib/restricted/abseil-cpp/absl/log/log_entry.cc index 19c3b3f1be..fe58a576b5 100644 --- a/contrib/restricted/abseil-cpp/absl/log/log_entry.cc +++ b/contrib/restricted/abseil-cpp/absl/log/log_entry.cc @@ -25,5 +25,17 @@ constexpr int LogEntry::kNoVerbosityLevel; constexpr int LogEntry::kNoVerboseLevel; #endif +// https://github.com/abseil/abseil-cpp/issues/1465 +// CMake builds on Apple platforms error when libraries are empty. +// Our CMake configuration can avoid this error on header-only libraries, +// but since this library is conditionally empty, including a single +// variable is an easy workaround. +#ifdef __APPLE__ +namespace log_internal { +extern const char kAvoidEmptyLogEntryLibraryWarning; +const char kAvoidEmptyLogEntryLibraryWarning = 0; +} // namespace log_internal +#endif // __APPLE__ + ABSL_NAMESPACE_END } // namespace absl diff --git a/contrib/restricted/abseil-cpp/absl/log/log_entry.h b/contrib/restricted/abseil-cpp/absl/log/log_entry.h index 9e4ae8eb76..7a55dfe29e 100644 --- a/contrib/restricted/abseil-cpp/absl/log/log_entry.h +++ b/contrib/restricted/abseil-cpp/absl/log/log_entry.h @@ -96,7 +96,8 @@ class LogEntry final { // LogEntry::verbosity() // // Returns this entry's verbosity, or `kNoVerbosityLevel` for a non-verbose - // entry. Verbosity control is not available outside of Google yet. + // entry. Taken from the argument to `VLOG` or from + // `LOG(...).WithVerbosity(...)`. int verbosity() const { return verbose_level_; } // LogEntry::timestamp() diff --git a/contrib/restricted/abseil-cpp/absl/log/vlog_is_on.h b/contrib/restricted/abseil-cpp/absl/log/vlog_is_on.h new file mode 100644 index 0000000000..7898651380 --- /dev/null +++ b/contrib/restricted/abseil-cpp/absl/log/vlog_is_on.h @@ -0,0 +1,72 @@ +// Copyright 2022 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: log/vlog_is_on.h +// ----------------------------------------------------------------------------- +// +// This header defines the `VLOG_IS_ON()` macro that controls the +// variable-verbosity conditional logging. +// +// It's used by `VLOG` in log.h, or it can also be used directly like this: +// +// if (VLOG_IS_ON(2)) { +// foo_server.RecomputeStatisticsExpensive(); +// LOG(INFO) << foo_server.LastStatisticsAsString(); +// } +// +// Each source file has an effective verbosity level that's a non-negative +// integer computed from the `--vmodule` and `--v` flags. +// `VLOG_IS_ON(n)` is true, and `VLOG(n)` logs, if that effective verbosity +// level is greater than or equal to `n`. +// +// `--vmodule` takes a comma-delimited list of key=value pairs. Each key is a +// pattern matched against filenames, and the values give the effective severity +// level applied to matching files. '?' and '*' characters in patterns are +// interpreted as single-character and zero-or-more-character wildcards. +// Patterns including a slash character are matched against full pathnames, +// while those without are matched against basenames only. One suffix (i.e. the +// last . and everything after it) is stripped from each filename prior to +// matching, as is the special suffix "-inl". +// +// Files are matched against globs in `--vmodule` in order, and the first match +// determines the verbosity level. +// +// Files which do not match any pattern in `--vmodule` use the value of `--v` as +// their effective verbosity level. The default is 0. +// +// SetVLOGLevel helper function is provided to do limited dynamic control over +// V-logging by appending to `--vmodule`. Because these go at the beginning of +// the list, they take priority over any globs previously added. +// +// Resetting --vmodule will override all previous modifications to `--vmodule`, +// including via SetVLOGLevel. + +#ifndef ABSL_LOG_VLOG_IS_ON_H_ +#define ABSL_LOG_VLOG_IS_ON_H_ + +#include "absl/log/absl_vlog_is_on.h" // IWYU pragma: export + +// IWYU pragma: private, include "absl/log/log.h" + +// Each VLOG_IS_ON call site gets its own VLogSite that registers with the +// global linked list of sites to asynchronously update its verbosity level on +// changes to --v or --vmodule. The verbosity can also be set by manually +// calling SetVLOGLevel. +// +// VLOG_IS_ON is not async signal safe, but it is guaranteed not to allocate +// new memory. +#define VLOG_IS_ON(verbose_level) ABSL_VLOG_IS_ON(verbose_level) + +#endif // ABSL_LOG_VLOG_IS_ON_H_ diff --git a/contrib/restricted/abseil-cpp/absl/log/ya.make b/contrib/restricted/abseil-cpp/absl/log/ya.make index cb73fdb2b2..f35baf8a56 100644 --- a/contrib/restricted/abseil-cpp/absl/log/ya.make +++ b/contrib/restricted/abseil-cpp/absl/log/ya.make @@ -35,12 +35,14 @@ SRCS( initialize.cc internal/check_op.cc internal/conditions.cc + internal/fnmatch.cc internal/globals.cc internal/log_format.cc internal/log_message.cc internal/log_sink_set.cc internal/nullguard.cc internal/proto.cc + internal/vlog_config.cc log_entry.cc log_sink.cc ) diff --git a/contrib/restricted/abseil-cpp/absl/memory/ya.make b/contrib/restricted/abseil-cpp/absl/memory/ya.make index 2d3aab1c79..4f191bfa0b 100644 --- a/contrib/restricted/abseil-cpp/absl/memory/ya.make +++ b/contrib/restricted/abseil-cpp/absl/memory/ya.make @@ -6,9 +6,9 @@ LICENSE(Apache-2.0) LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(20230802.1) +VERSION(20240116.0) -ORIGINAL_SOURCE(https://github.com/abseil/abseil-cpp/archive/20230802.1.tar.gz) +ORIGINAL_SOURCE(https://github.com/abseil/abseil-cpp/archive/20240116.0.tar.gz) PEERDIR( contrib/restricted/abseil-cpp/absl/meta diff --git a/contrib/restricted/abseil-cpp/absl/meta/ya.make b/contrib/restricted/abseil-cpp/absl/meta/ya.make index 39b472c0fa..8a0016666e 100644 --- a/contrib/restricted/abseil-cpp/absl/meta/ya.make +++ b/contrib/restricted/abseil-cpp/absl/meta/ya.make @@ -6,9 +6,9 @@ LICENSE(Apache-2.0) LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(20230802.1) +VERSION(20240116.0) -ORIGINAL_SOURCE(https://github.com/abseil/abseil-cpp/archive/20230802.1.tar.gz) +ORIGINAL_SOURCE(https://github.com/abseil/abseil-cpp/archive/20240116.0.tar.gz) PEERDIR( contrib/restricted/abseil-cpp/absl/base diff --git a/contrib/restricted/abseil-cpp/absl/numeric/bits.h b/contrib/restricted/abseil-cpp/absl/numeric/bits.h index 5ed36f5296..c76454c8fb 100644 --- a/contrib/restricted/abseil-cpp/absl/numeric/bits.h +++ b/contrib/restricted/abseil-cpp/absl/numeric/bits.h @@ -49,9 +49,19 @@ namespace absl { ABSL_NAMESPACE_BEGIN -#if !(defined(__cpp_lib_bitops) && __cpp_lib_bitops >= 201907L) -// rotating +// https://github.com/llvm/llvm-project/issues/64544 +// libc++ had the wrong signature for std::rotl and std::rotr +// prior to libc++ 18.0. +// +#if (defined(__cpp_lib_bitops) && __cpp_lib_bitops >= 201907L) && \ + (!defined(_LIBCPP_VERSION) || _LIBCPP_VERSION >= 180000) +using std::rotl; +using std::rotr; + +#else + +// Rotating functions template <class T> ABSL_MUST_USE_RESULT constexpr typename std::enable_if<std::is_unsigned<T>::value, T>::type @@ -66,6 +76,22 @@ ABSL_MUST_USE_RESULT constexpr return numeric_internal::RotateRight(x, s); } +#endif + +// https://github.com/llvm/llvm-project/issues/64544 +// libc++ had the wrong signature for std::rotl and std::rotr +// prior to libc++ 18.0. +// +#if (defined(__cpp_lib_bitops) && __cpp_lib_bitops >= 201907L) + +using std::countl_one; +using std::countl_zero; +using std::countr_one; +using std::countr_zero; +using std::popcount; + +#else + // Counting functions // // While these functions are typically constexpr, on some platforms, they may @@ -107,19 +133,18 @@ ABSL_INTERNAL_CONSTEXPR_POPCOUNT inline popcount(T x) noexcept { return numeric_internal::Popcount(x); } -#else // defined(__cpp_lib_bitops) && __cpp_lib_bitops >= 201907L - -using std::countl_one; -using std::countl_zero; -using std::countr_one; -using std::countr_zero; -using std::popcount; -using std::rotl; -using std::rotr; #endif -#if !(defined(__cpp_lib_int_pow2) && __cpp_lib_int_pow2 >= 202002L) +#if (defined(__cpp_lib_int_pow2) && __cpp_lib_int_pow2 >= 202002L) + +using std::bit_ceil; +using std::bit_floor; +using std::bit_width; +using std::has_single_bit; + +#else + // Returns: true if x is an integral power of two; false otherwise. template <class T> constexpr inline typename std::enable_if<std::is_unsigned<T>::value, bool>::type @@ -162,12 +187,6 @@ ABSL_INTERNAL_CONSTEXPR_CLZ inline return has_single_bit(x) ? T{1} << (bit_width(x) - 1) : numeric_internal::BitCeilNonPowerOf2(x); } -#else // defined(__cpp_lib_int_pow2) && __cpp_lib_int_pow2 >= 202002L - -using std::bit_ceil; -using std::bit_floor; -using std::bit_width; -using std::has_single_bit; #endif diff --git a/contrib/restricted/abseil-cpp/absl/status/internal/status_internal.cc b/contrib/restricted/abseil-cpp/absl/status/internal/status_internal.cc new file mode 100644 index 0000000000..a91567549e --- /dev/null +++ b/contrib/restricted/abseil-cpp/absl/status/internal/status_internal.cc @@ -0,0 +1,248 @@ +// Copyright 2023 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/status/internal/status_internal.h" + +#include <atomic> +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <cstdio> +#include <cstring> +#include <memory> +#include <string> +#include <utility> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/macros.h" +#include "absl/base/nullability.h" +#include "absl/debugging/stacktrace.h" +#include "absl/debugging/symbolize.h" +#include "absl/memory/memory.h" +#include "absl/status/status.h" +#include "absl/status/status_payload_printer.h" +#include "absl/strings/cord.h" +#include "absl/strings/escaping.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace status_internal { + +void StatusRep::Unref() const { + // Fast path: if ref==1, there is no need for a RefCountDec (since + // this is the only reference and therefore no other thread is + // allowed to be mucking with r). + if (ref_.load(std::memory_order_acquire) == 1 || + ref_.fetch_sub(1, std::memory_order_acq_rel) - 1 == 0) { + delete this; + } +} + +static absl::optional<size_t> FindPayloadIndexByUrl( + const Payloads* payloads, absl::string_view type_url) { + if (payloads == nullptr) return absl::nullopt; + + for (size_t i = 0; i < payloads->size(); ++i) { + if ((*payloads)[i].type_url == type_url) return i; + } + + return absl::nullopt; +} + +absl::optional<absl::Cord> StatusRep::GetPayload( + absl::string_view type_url) const { + absl::optional<size_t> index = + status_internal::FindPayloadIndexByUrl(payloads_.get(), type_url); + if (index.has_value()) return (*payloads_)[index.value()].payload; + + return absl::nullopt; +} + +void StatusRep::SetPayload(absl::string_view type_url, absl::Cord payload) { + if (payloads_ == nullptr) { + payloads_ = absl::make_unique<status_internal::Payloads>(); + } + + absl::optional<size_t> index = + status_internal::FindPayloadIndexByUrl(payloads_.get(), type_url); + if (index.has_value()) { + (*payloads_)[index.value()].payload = std::move(payload); + return; + } + + payloads_->push_back({std::string(type_url), std::move(payload)}); +} + +StatusRep::EraseResult StatusRep::ErasePayload(absl::string_view type_url) { + absl::optional<size_t> index = + status_internal::FindPayloadIndexByUrl(payloads_.get(), type_url); + if (!index.has_value()) return {false, Status::PointerToRep(this)}; + payloads_->erase(payloads_->begin() + index.value()); + if (payloads_->empty() && message_.empty()) { + // Special case: If this can be represented inlined, it MUST be inlined + // (== depends on this behavior). + EraseResult result = {true, Status::CodeToInlinedRep(code_)}; + Unref(); + return result; + } + return {true, Status::PointerToRep(this)}; +} + +void StatusRep::ForEachPayload( + absl::FunctionRef<void(absl::string_view, const absl::Cord&)> visitor) + const { + if (auto* payloads = payloads_.get()) { + bool in_reverse = + payloads->size() > 1 && reinterpret_cast<uintptr_t>(payloads) % 13 > 6; + + for (size_t index = 0; index < payloads->size(); ++index) { + const auto& elem = + (*payloads)[in_reverse ? payloads->size() - 1 - index : index]; + +#ifdef NDEBUG + visitor(elem.type_url, elem.payload); +#else + // In debug mode invalidate the type url to prevent users from relying on + // this string lifetime. + + // NOLINTNEXTLINE intentional extra conversion to force temporary. + visitor(std::string(elem.type_url), elem.payload); +#endif // NDEBUG + } + } +} + +std::string StatusRep::ToString(StatusToStringMode mode) const { + std::string text; + absl::StrAppend(&text, absl::StatusCodeToString(code()), ": ", message()); + + const bool with_payload = (mode & StatusToStringMode::kWithPayload) == + StatusToStringMode::kWithPayload; + + if (with_payload) { + status_internal::StatusPayloadPrinter printer = + status_internal::GetStatusPayloadPrinter(); + this->ForEachPayload([&](absl::string_view type_url, + const absl::Cord& payload) { + absl::optional<std::string> result; + if (printer) result = printer(type_url, payload); + absl::StrAppend( + &text, " [", type_url, "='", + result.has_value() ? *result : absl::CHexEscape(std::string(payload)), + "']"); + }); + } + + return text; +} + +bool StatusRep::operator==(const StatusRep& other) const { + assert(this != &other); + if (code_ != other.code_) return false; + if (message_ != other.message_) return false; + const status_internal::Payloads* this_payloads = payloads_.get(); + const status_internal::Payloads* other_payloads = other.payloads_.get(); + + const status_internal::Payloads no_payloads; + const status_internal::Payloads* larger_payloads = + this_payloads ? this_payloads : &no_payloads; + const status_internal::Payloads* smaller_payloads = + other_payloads ? other_payloads : &no_payloads; + if (larger_payloads->size() < smaller_payloads->size()) { + std::swap(larger_payloads, smaller_payloads); + } + if ((larger_payloads->size() - smaller_payloads->size()) > 1) return false; + // Payloads can be ordered differently, so we can't just compare payload + // vectors. + for (const auto& payload : *larger_payloads) { + + bool found = false; + for (const auto& other_payload : *smaller_payloads) { + if (payload.type_url == other_payload.type_url) { + if (payload.payload != other_payload.payload) { + return false; + } + found = true; + break; + } + } + if (!found) return false; + } + return true; +} + +absl::Nonnull<StatusRep*> StatusRep::CloneAndUnref() const { + // Optimization: no need to create a clone if we already have a refcount of 1. + if (ref_.load(std::memory_order_acquire) == 1) { + // All StatusRep instances are heap allocated and mutable, therefore this + // const_cast will never cast away const from a stack instance. + // + // CloneAndUnref is the only method that doesn't involve an external cast to + // get a mutable StatusRep* from the uintptr_t rep stored in Status. + return const_cast<StatusRep*>(this); + } + std::unique_ptr<status_internal::Payloads> payloads; + if (payloads_) { + payloads = absl::make_unique<status_internal::Payloads>(*payloads_); + } + auto* new_rep = new StatusRep(code_, message_, std::move(payloads)); + Unref(); + return new_rep; +} + +// Convert canonical code to a value known to this binary. +absl::StatusCode MapToLocalCode(int value) { + absl::StatusCode code = static_cast<absl::StatusCode>(value); + switch (code) { + case absl::StatusCode::kOk: + case absl::StatusCode::kCancelled: + case absl::StatusCode::kUnknown: + case absl::StatusCode::kInvalidArgument: + case absl::StatusCode::kDeadlineExceeded: + case absl::StatusCode::kNotFound: + case absl::StatusCode::kAlreadyExists: + case absl::StatusCode::kPermissionDenied: + case absl::StatusCode::kResourceExhausted: + case absl::StatusCode::kFailedPrecondition: + case absl::StatusCode::kAborted: + case absl::StatusCode::kOutOfRange: + case absl::StatusCode::kUnimplemented: + case absl::StatusCode::kInternal: + case absl::StatusCode::kUnavailable: + case absl::StatusCode::kDataLoss: + case absl::StatusCode::kUnauthenticated: + return code; + default: + return absl::StatusCode::kUnknown; + } +} + +absl::Nonnull<std::string*> MakeCheckFailString( + absl::Nonnull<const absl::Status*> status, + absl::Nonnull<const char*> prefix) { + return new std::string( + absl::StrCat(prefix, " (", + status->ToString(StatusToStringMode::kWithEverything), ")")); +} + +} // namespace status_internal + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/contrib/restricted/abseil-cpp/absl/status/internal/status_internal.h b/contrib/restricted/abseil-cpp/absl/status/internal/status_internal.h index 6198e726b3..c9f4383272 100644 --- a/contrib/restricted/abseil-cpp/absl/status/internal/status_internal.h +++ b/contrib/restricted/abseil-cpp/absl/status/internal/status_internal.h @@ -14,13 +14,19 @@ #ifndef ABSL_STATUS_INTERNAL_STATUS_INTERNAL_H_ #define ABSL_STATUS_INTERNAL_STATUS_INTERNAL_H_ +#include <atomic> +#include <cstdint> #include <memory> #include <string> #include <utility> #include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/nullability.h" #include "absl/container/inlined_vector.h" #include "absl/strings/cord.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" #ifndef SWIG // Disabled for SWIG as it doesn't parse attributes correctly. @@ -32,9 +38,9 @@ ABSL_NAMESPACE_BEGIN // TODO(b/176172494): ABSL_MUST_USE_RESULT should expand to the more strict // [[nodiscard]]. For now, just use [[nodiscard]] directly when it is available. #if ABSL_HAVE_CPP_ATTRIBUTE(nodiscard) -class [[nodiscard]] Status; +class [[nodiscard]] ABSL_ATTRIBUTE_TRIVIAL_ABI Status; #else -class ABSL_MUST_USE_RESULT Status; +class ABSL_MUST_USE_RESULT ABSL_ATTRIBUTE_TRIVIAL_ABI Status; #endif ABSL_NAMESPACE_END } // namespace absl @@ -44,6 +50,7 @@ namespace absl { ABSL_NAMESPACE_BEGIN enum class StatusCode : int; +enum class StatusToStringMode : int; namespace status_internal { @@ -56,22 +63,54 @@ struct Payload { using Payloads = absl::InlinedVector<Payload, 1>; // Reference-counted representation of Status data. -struct StatusRep { +class StatusRep { + public: StatusRep(absl::StatusCode code_arg, absl::string_view message_arg, std::unique_ptr<status_internal::Payloads> payloads_arg) - : ref(int32_t{1}), - code(code_arg), - message(message_arg), - payloads(std::move(payloads_arg)) {} - - std::atomic<int32_t> ref; - absl::StatusCode code; + : ref_(int32_t{1}), + code_(code_arg), + message_(message_arg), + payloads_(std::move(payloads_arg)) {} + + absl::StatusCode code() const { return code_; } + const std::string& message() const { return message_; } + + // Ref and unref are const to allow access through a const pointer, and are + // used during copying operations. + void Ref() const { ref_.fetch_add(1, std::memory_order_relaxed); } + void Unref() const; + + // Payload methods correspond to the same methods in absl::Status. + absl::optional<absl::Cord> GetPayload(absl::string_view type_url) const; + void SetPayload(absl::string_view type_url, absl::Cord payload); + struct EraseResult { + bool erased; + uintptr_t new_rep; + }; + EraseResult ErasePayload(absl::string_view type_url); + void ForEachPayload( + absl::FunctionRef<void(absl::string_view, const absl::Cord&)> visitor) + const; + + std::string ToString(StatusToStringMode mode) const; + + bool operator==(const StatusRep& other) const; + bool operator!=(const StatusRep& other) const { return !(*this == other); } + + // Returns an equivalent heap allocated StatusRep with refcount 1. + // + // `this` is not safe to be used after calling as it may have been deleted. + absl::Nonnull<StatusRep*> CloneAndUnref() const; + + private: + mutable std::atomic<int32_t> ref_; + absl::StatusCode code_; // As an internal implementation detail, we guarantee that if status.message() // is non-empty, then the resulting string_view is null terminated. // This is required to implement 'StatusMessageAsCStr(...)' - std::string message; - std::unique_ptr<status_internal::Payloads> payloads; + std::string message_; + std::unique_ptr<status_internal::Payloads> payloads_; }; absl::StatusCode MapToLocalCode(int value); @@ -80,8 +119,10 @@ absl::StatusCode MapToLocalCode(int value); // suitable for output as an error message in assertion/`CHECK()` failures. // // This is an internal implementation detail for Abseil logging. -std::string* MakeCheckFailString(const absl::Status* status, - const char* prefix); +ABSL_ATTRIBUTE_PURE_FUNCTION +absl::Nonnull<std::string*> MakeCheckFailString( + absl::Nonnull<const absl::Status*> status, + absl::Nonnull<const char*> prefix); } // namespace status_internal diff --git a/contrib/restricted/abseil-cpp/absl/status/internal/statusor_internal.h b/contrib/restricted/abseil-cpp/absl/status/internal/statusor_internal.h index 49cead7a7f..5be94903b7 100644 --- a/contrib/restricted/abseil-cpp/absl/status/internal/statusor_internal.h +++ b/contrib/restricted/abseil-cpp/absl/status/internal/statusor_internal.h @@ -14,12 +14,15 @@ #ifndef ABSL_STATUS_INTERNAL_STATUSOR_INTERNAL_H_ #define ABSL_STATUS_INTERNAL_STATUSOR_INTERNAL_H_ +#include <cstdint> #include <type_traits> #include <utility> #include "absl/base/attributes.h" +#include "absl/base/nullability.h" #include "absl/meta/type_traits.h" #include "absl/status/status.h" +#include "absl/strings/string_view.h" #include "absl/utility/utility.h" namespace absl { @@ -123,7 +126,7 @@ using IsForwardingAssignmentValid = absl::disjunction< class Helper { public: // Move type-agnostic error handling to the .cc. - static void HandleInvalidStatusCtorArg(Status*); + static void HandleInvalidStatusCtorArg(absl::Nonnull<Status*>); ABSL_ATTRIBUTE_NORETURN static void Crash(const absl::Status& status); }; @@ -131,7 +134,8 @@ class Helper { // the constructor. // This abstraction is here mostly for the gcc performance fix. template <typename T, typename... Args> -ABSL_ATTRIBUTE_NONNULL(1) void PlacementNew(void* p, Args&&... args) { +ABSL_ATTRIBUTE_NONNULL(1) +void PlacementNew(absl::Nonnull<void*> p, Args&&... args) { new (p) T(std::forward<Args>(args)...); } @@ -377,6 +381,53 @@ struct MoveAssignBase<T, false> { ABSL_ATTRIBUTE_NORETURN void ThrowBadStatusOrAccess(absl::Status status); +// Used to introduce jitter into the output of printing functions for +// `StatusOr` (i.e. `AbslStringify` and `operator<<`). +class StringifyRandom { + enum BracesType { + kBareParens = 0, + kSpaceParens, + kBareBrackets, + kSpaceBrackets, + }; + + // Returns a random `BracesType` determined once per binary load. + static BracesType RandomBraces() { + static const BracesType kRandomBraces = static_cast<BracesType>( + (reinterpret_cast<uintptr_t>(&kRandomBraces) >> 4) % 4); + return kRandomBraces; + } + + public: + static inline absl::string_view OpenBrackets() { + switch (RandomBraces()) { + case kBareParens: + return "("; + case kSpaceParens: + return "( "; + case kBareBrackets: + return "["; + case kSpaceBrackets: + return "[ "; + } + return "("; + } + + static inline absl::string_view CloseBrackets() { + switch (RandomBraces()) { + case kBareParens: + return ")"; + case kSpaceParens: + return " )"; + case kBareBrackets: + return "]"; + case kSpaceBrackets: + return " ]"; + } + return ")"; + } +}; + } // namespace internal_statusor ABSL_NAMESPACE_END } // namespace absl diff --git a/contrib/restricted/abseil-cpp/absl/status/status.cc b/contrib/restricted/abseil-cpp/absl/status/status.cc index 26e68294ac..4dd5ae06ce 100644 --- a/contrib/restricted/abseil-cpp/absl/status/status.cc +++ b/contrib/restricted/abseil-cpp/absl/status/status.cc @@ -15,23 +15,37 @@ #include <errno.h> -#include <cassert> -#include <utility> - +#include <atomic> +#include <cstddef> +#include <cstdint> +#include <cstring> +#include <memory> +#include <ostream> +#include <string> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" #include "absl/base/internal/raw_logging.h" #include "absl/base/internal/strerror.h" #include "absl/base/macros.h" +#include "absl/base/no_destructor.h" +#include "absl/base/nullability.h" #include "absl/debugging/stacktrace.h" #include "absl/debugging/symbolize.h" -#include "absl/status/status_payload_printer.h" -#include "absl/strings/escaping.h" +#include "absl/status/internal/status_internal.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" namespace absl { ABSL_NAMESPACE_BEGIN +static_assert( + alignof(status_internal::StatusRep) >= 4, + "absl::Status assumes it can use the bottom 2 bits of a StatusRep*."); + std::string StatusCodeToString(StatusCode code) { switch (code) { case StatusCode::kOk: @@ -77,146 +91,18 @@ std::ostream& operator<<(std::ostream& os, StatusCode code) { return os << StatusCodeToString(code); } -namespace status_internal { - -static absl::optional<size_t> FindPayloadIndexByUrl( - const Payloads* payloads, absl::string_view type_url) { - if (payloads == nullptr) return absl::nullopt; - - for (size_t i = 0; i < payloads->size(); ++i) { - if ((*payloads)[i].type_url == type_url) return i; - } - - return absl::nullopt; -} - -// Convert canonical code to a value known to this binary. -absl::StatusCode MapToLocalCode(int value) { - absl::StatusCode code = static_cast<absl::StatusCode>(value); - switch (code) { - case absl::StatusCode::kOk: - case absl::StatusCode::kCancelled: - case absl::StatusCode::kUnknown: - case absl::StatusCode::kInvalidArgument: - case absl::StatusCode::kDeadlineExceeded: - case absl::StatusCode::kNotFound: - case absl::StatusCode::kAlreadyExists: - case absl::StatusCode::kPermissionDenied: - case absl::StatusCode::kResourceExhausted: - case absl::StatusCode::kFailedPrecondition: - case absl::StatusCode::kAborted: - case absl::StatusCode::kOutOfRange: - case absl::StatusCode::kUnimplemented: - case absl::StatusCode::kInternal: - case absl::StatusCode::kUnavailable: - case absl::StatusCode::kDataLoss: - case absl::StatusCode::kUnauthenticated: - return code; - default: - return absl::StatusCode::kUnknown; - } -} -} // namespace status_internal - -absl::optional<absl::Cord> Status::GetPayload( - absl::string_view type_url) const { - const auto* payloads = GetPayloads(); - absl::optional<size_t> index = - status_internal::FindPayloadIndexByUrl(payloads, type_url); - if (index.has_value()) return (*payloads)[index.value()].payload; - - return absl::nullopt; -} - -void Status::SetPayload(absl::string_view type_url, absl::Cord payload) { - if (ok()) return; - - PrepareToModify(); - - status_internal::StatusRep* rep = RepToPointer(rep_); - if (!rep->payloads) { - rep->payloads = absl::make_unique<status_internal::Payloads>(); - } - - absl::optional<size_t> index = - status_internal::FindPayloadIndexByUrl(rep->payloads.get(), type_url); - if (index.has_value()) { - (*rep->payloads)[index.value()].payload = std::move(payload); - return; - } - - rep->payloads->push_back({std::string(type_url), std::move(payload)}); -} - -bool Status::ErasePayload(absl::string_view type_url) { - absl::optional<size_t> index = - status_internal::FindPayloadIndexByUrl(GetPayloads(), type_url); - if (index.has_value()) { - PrepareToModify(); - GetPayloads()->erase(GetPayloads()->begin() + index.value()); - if (GetPayloads()->empty() && message().empty()) { - // Special case: If this can be represented inlined, it MUST be - // inlined (EqualsSlow depends on this behavior). - StatusCode c = static_cast<StatusCode>(raw_code()); - Unref(rep_); - rep_ = CodeToInlinedRep(c); - } - return true; - } - - return false; -} - -void Status::ForEachPayload( - absl::FunctionRef<void(absl::string_view, const absl::Cord&)> visitor) - const { - if (auto* payloads = GetPayloads()) { - bool in_reverse = - payloads->size() > 1 && reinterpret_cast<uintptr_t>(payloads) % 13 > 6; - - for (size_t index = 0; index < payloads->size(); ++index) { - const auto& elem = - (*payloads)[in_reverse ? payloads->size() - 1 - index : index]; - -#ifdef NDEBUG - visitor(elem.type_url, elem.payload); -#else - // In debug mode invalidate the type url to prevent users from relying on - // this string lifetime. - - // NOLINTNEXTLINE intentional extra conversion to force temporary. - visitor(std::string(elem.type_url), elem.payload); -#endif // NDEBUG - } - } -} - -const std::string* Status::EmptyString() { - static union EmptyString { - std::string str; - ~EmptyString() {} - } empty = {{}}; - return &empty.str; +absl::Nonnull<const std::string*> Status::EmptyString() { + static const absl::NoDestructor<std::string> kEmpty; + return kEmpty.get(); } #ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL constexpr const char Status::kMovedFromString[]; #endif -const std::string* Status::MovedFromString() { - static std::string* moved_from_string = new std::string(kMovedFromString); - return moved_from_string; -} - -void Status::UnrefNonInlined(uintptr_t rep) { - status_internal::StatusRep* r = RepToPointer(rep); - // Fast path: if ref==1, there is no need for a RefCountDec (since - // this is the only reference and therefore no other thread is - // allowed to be mucking with r). - if (r->ref.load(std::memory_order_acquire) == 1 || - r->ref.fetch_sub(1, std::memory_order_acq_rel) - 1 == 0) { - delete r; - } +absl::Nonnull<const std::string*> Status::MovedFromString() { + static const absl::NoDestructor<std::string> kMovedFrom(kMovedFromString); + return kMovedFrom.get(); } Status::Status(absl::StatusCode code, absl::string_view msg) @@ -226,97 +112,20 @@ Status::Status(absl::StatusCode code, absl::string_view msg) } } -int Status::raw_code() const { - if (IsInlined(rep_)) { - return static_cast<int>(InlinedRepToCode(rep_)); +absl::Nonnull<status_internal::StatusRep*> Status::PrepareToModify( + uintptr_t rep) { + if (IsInlined(rep)) { + return new status_internal::StatusRep(InlinedRepToCode(rep), + absl::string_view(), nullptr); } - status_internal::StatusRep* rep = RepToPointer(rep_); - return static_cast<int>(rep->code); + return RepToPointer(rep)->CloneAndUnref(); } -absl::StatusCode Status::code() const { - return status_internal::MapToLocalCode(raw_code()); -} - -void Status::PrepareToModify() { - ABSL_RAW_CHECK(!ok(), "PrepareToModify shouldn't be called on OK status."); - if (IsInlined(rep_)) { - rep_ = PointerToRep(new status_internal::StatusRep( - static_cast<absl::StatusCode>(raw_code()), absl::string_view(), - nullptr)); - return; +std::string Status::ToStringSlow(uintptr_t rep, StatusToStringMode mode) { + if (IsInlined(rep)) { + return absl::StrCat(absl::StatusCodeToString(InlinedRepToCode(rep)), ": "); } - - uintptr_t rep_i = rep_; - status_internal::StatusRep* rep = RepToPointer(rep_); - if (rep->ref.load(std::memory_order_acquire) != 1) { - std::unique_ptr<status_internal::Payloads> payloads; - if (rep->payloads) { - payloads = absl::make_unique<status_internal::Payloads>(*rep->payloads); - } - status_internal::StatusRep* const new_rep = new status_internal::StatusRep( - rep->code, message(), std::move(payloads)); - rep_ = PointerToRep(new_rep); - UnrefNonInlined(rep_i); - } -} - -bool Status::EqualsSlow(const absl::Status& a, const absl::Status& b) { - if (IsInlined(a.rep_) != IsInlined(b.rep_)) return false; - if (a.message() != b.message()) return false; - if (a.raw_code() != b.raw_code()) return false; - if (a.GetPayloads() == b.GetPayloads()) return true; - - const status_internal::Payloads no_payloads; - const status_internal::Payloads* larger_payloads = - a.GetPayloads() ? a.GetPayloads() : &no_payloads; - const status_internal::Payloads* smaller_payloads = - b.GetPayloads() ? b.GetPayloads() : &no_payloads; - if (larger_payloads->size() < smaller_payloads->size()) { - std::swap(larger_payloads, smaller_payloads); - } - if ((larger_payloads->size() - smaller_payloads->size()) > 1) return false; - // Payloads can be ordered differently, so we can't just compare payload - // vectors. - for (const auto& payload : *larger_payloads) { - - bool found = false; - for (const auto& other_payload : *smaller_payloads) { - if (payload.type_url == other_payload.type_url) { - if (payload.payload != other_payload.payload) { - return false; - } - found = true; - break; - } - } - if (!found) return false; - } - return true; -} - -std::string Status::ToStringSlow(StatusToStringMode mode) const { - std::string text; - absl::StrAppend(&text, absl::StatusCodeToString(code()), ": ", message()); - - const bool with_payload = (mode & StatusToStringMode::kWithPayload) == - StatusToStringMode::kWithPayload; - - if (with_payload) { - status_internal::StatusPayloadPrinter printer = - status_internal::GetStatusPayloadPrinter(); - this->ForEachPayload([&](absl::string_view type_url, - const absl::Cord& payload) { - absl::optional<std::string> result; - if (printer) result = printer(type_url, payload); - absl::StrAppend( - &text, " [", type_url, "='", - result.has_value() ? *result : absl::CHexEscape(std::string(payload)), - "']"); - }); - } - - return text; + return RepToPointer(rep)->ToString(mode); } std::ostream& operator<<(std::ostream& os, const Status& x) { @@ -605,18 +414,7 @@ Status ErrnoToStatus(int error_number, absl::string_view message) { MessageForErrnoToStatus(error_number, message)); } -namespace status_internal { - -std::string* MakeCheckFailString(const absl::Status* status, - const char* prefix) { - return new std::string( - absl::StrCat(prefix, " (", - status->ToString(StatusToStringMode::kWithEverything), ")")); -} - -} // namespace status_internal - -const char* StatusMessageAsCStr(const Status& status) { +absl::Nonnull<const char*> StatusMessageAsCStr(const Status& status) { // As an internal implementation detail, we guarantee that if status.message() // is non-empty, then the resulting string_view is null terminated. auto sv_message = status.message(); diff --git a/contrib/restricted/abseil-cpp/absl/status/status.h b/contrib/restricted/abseil-cpp/absl/status/status.h index 595064c0f1..9ce16db961 100644 --- a/contrib/restricted/abseil-cpp/absl/status/status.h +++ b/contrib/restricted/abseil-cpp/absl/status/status.h @@ -51,10 +51,17 @@ #ifndef ABSL_STATUS_STATUS_H_ #define ABSL_STATUS_STATUS_H_ +#include <cassert> +#include <cstdint> #include <ostream> #include <string> #include <utility> +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/macros.h" +#include "absl/base/nullability.h" +#include "absl/base/optimization.h" #include "absl/functional/function_ref.h" #include "absl/status/internal/status_internal.h" #include "absl/strings/cord.h" @@ -421,7 +428,7 @@ inline StatusToStringMode& operator^=(StatusToStringMode& lhs, // Returned Status objects may not be ignored. status_internal.h has a forward // declaration of the form // class ABSL_MUST_USE_RESULT Status; -class Status final { +class ABSL_ATTRIBUTE_TRIVIAL_ABI Status final { public: // Constructors @@ -516,6 +523,12 @@ class Status final { std::string ToString( StatusToStringMode mode = StatusToStringMode::kDefault) const; + // Support `absl::StrCat`, `absl::StrFormat`, etc. + template <typename Sink> + friend void AbslStringify(Sink& sink, const Status& status) { + sink.Append(status.ToString(StatusToStringMode::kWithEverything)); + } + // Status::IgnoreError() // // Ignores any errors. This method does nothing except potentially suppress @@ -602,56 +615,57 @@ class Status final { // code, and an empty error message. explicit Status(absl::StatusCode code); - static void UnrefNonInlined(uintptr_t rep); + // Underlying constructor for status from a rep_. + explicit Status(uintptr_t rep) : rep_(rep) {} + static void Ref(uintptr_t rep); static void Unref(uintptr_t rep); // REQUIRES: !ok() - // Ensures rep_ is not shared with any other Status. - void PrepareToModify(); - - const status_internal::Payloads* GetPayloads() const; - status_internal::Payloads* GetPayloads(); - - static bool EqualsSlow(const absl::Status& a, const absl::Status& b); + // Ensures rep is not inlined or shared with any other Status. + static absl::Nonnull<status_internal::StatusRep*> PrepareToModify( + uintptr_t rep); // MSVC 14.0 limitation requires the const. static constexpr const char kMovedFromString[] = "Status accessed after move."; - static const std::string* EmptyString(); - static const std::string* MovedFromString(); + static absl::Nonnull<const std::string*> EmptyString(); + static absl::Nonnull<const std::string*> MovedFromString(); // Returns whether rep contains an inlined representation. // See rep_ for details. - static bool IsInlined(uintptr_t rep); + static constexpr bool IsInlined(uintptr_t rep); // Indicates whether this Status was the rhs of a move operation. See rep_ // for details. - static bool IsMovedFrom(uintptr_t rep); - static uintptr_t MovedFromRep(); + static constexpr bool IsMovedFrom(uintptr_t rep); + static constexpr uintptr_t MovedFromRep(); // Convert between error::Code and the inlined uintptr_t representation used // by rep_. See rep_ for details. - static uintptr_t CodeToInlinedRep(absl::StatusCode code); - static absl::StatusCode InlinedRepToCode(uintptr_t rep); + static constexpr uintptr_t CodeToInlinedRep(absl::StatusCode code); + static constexpr absl::StatusCode InlinedRepToCode(uintptr_t rep); // Converts between StatusRep* and the external uintptr_t representation used // by rep_. See rep_ for details. static uintptr_t PointerToRep(status_internal::StatusRep* r); - static status_internal::StatusRep* RepToPointer(uintptr_t r); + static absl::Nonnull<const status_internal::StatusRep*> RepToPointer( + uintptr_t r); - std::string ToStringSlow(StatusToStringMode mode) const; + static std::string ToStringSlow(uintptr_t rep, StatusToStringMode mode); // Status supports two different representations. - // - When the low bit is off it is an inlined representation. + // - When the low bit is set it is an inlined representation. // It uses the canonical error space, no message or payload. // The error code is (rep_ >> 2). // The (rep_ & 2) bit is the "moved from" indicator, used in IsMovedFrom(). - // - When the low bit is on it is an external representation. + // - When the low bit is off it is an external representation. // In this case all the data comes from a heap allocated Rep object. - // (rep_ - 1) is a status_internal::StatusRep* pointer to that structure. + // rep_ is a status_internal::StatusRep* pointer to that structure. uintptr_t rep_; + + friend class status_internal::StatusRep; }; // OkStatus() @@ -755,11 +769,11 @@ Status ErrnoToStatus(int error_number, absl::string_view message); // Implementation details follow //------------------------------------------------------------------------------ -inline Status::Status() : rep_(CodeToInlinedRep(absl::StatusCode::kOk)) {} +inline Status::Status() : Status(absl::StatusCode::kOk) {} -inline Status::Status(absl::StatusCode code) : rep_(CodeToInlinedRep(code)) {} +inline Status::Status(absl::StatusCode code) : Status(CodeToInlinedRep(code)) {} -inline Status::Status(const Status& x) : rep_(x.rep_) { Ref(rep_); } +inline Status::Status(const Status& x) : Status(x.rep_) { Ref(rep_); } inline Status& Status::operator=(const Status& x) { uintptr_t old_rep = rep_; @@ -771,7 +785,7 @@ inline Status& Status::operator=(const Status& x) { return *this; } -inline Status::Status(Status&& x) noexcept : rep_(x.rep_) { +inline Status::Status(Status&& x) noexcept : Status(x.rep_) { x.rep_ = MovedFromRep(); } @@ -803,15 +817,27 @@ inline bool Status::ok() const { return rep_ == CodeToInlinedRep(absl::StatusCode::kOk); } +inline absl::StatusCode Status::code() const { + return status_internal::MapToLocalCode(raw_code()); +} + +inline int Status::raw_code() const { + if (IsInlined(rep_)) return static_cast<int>(InlinedRepToCode(rep_)); + return static_cast<int>(RepToPointer(rep_)->code()); +} + inline absl::string_view Status::message() const { return !IsInlined(rep_) - ? RepToPointer(rep_)->message + ? RepToPointer(rep_)->message() : (IsMovedFrom(rep_) ? absl::string_view(kMovedFromString) : absl::string_view()); } inline bool operator==(const Status& lhs, const Status& rhs) { - return lhs.rep_ == rhs.rep_ || Status::EqualsSlow(lhs, rhs); + if (lhs.rep_ == rhs.rep_) return true; + if (Status::IsInlined(lhs.rep_)) return false; + if (Status::IsInlined(rhs.rep_)) return false; + return *Status::RepToPointer(lhs.rep_) == *Status::RepToPointer(rhs.rep_); } inline bool operator!=(const Status& lhs, const Status& rhs) { @@ -819,7 +845,7 @@ inline bool operator!=(const Status& lhs, const Status& rhs) { } inline std::string Status::ToString(StatusToStringMode mode) const { - return ok() ? "OK" : ToStringSlow(mode); + return ok() ? "OK" : ToStringSlow(rep_, mode); } inline void Status::IgnoreError() const { @@ -831,52 +857,68 @@ inline void swap(absl::Status& a, absl::Status& b) { swap(a.rep_, b.rep_); } -inline const status_internal::Payloads* Status::GetPayloads() const { - return IsInlined(rep_) ? nullptr : RepToPointer(rep_)->payloads.get(); +inline absl::optional<absl::Cord> Status::GetPayload( + absl::string_view type_url) const { + if (IsInlined(rep_)) return absl::nullopt; + return RepToPointer(rep_)->GetPayload(type_url); } -inline status_internal::Payloads* Status::GetPayloads() { - return IsInlined(rep_) ? nullptr : RepToPointer(rep_)->payloads.get(); +inline void Status::SetPayload(absl::string_view type_url, absl::Cord payload) { + if (ok()) return; + status_internal::StatusRep* rep = PrepareToModify(rep_); + rep->SetPayload(type_url, std::move(payload)); + rep_ = PointerToRep(rep); } -inline bool Status::IsInlined(uintptr_t rep) { return (rep & 1) == 0; } - -inline bool Status::IsMovedFrom(uintptr_t rep) { - return IsInlined(rep) && (rep & 2) != 0; +inline bool Status::ErasePayload(absl::string_view type_url) { + if (IsInlined(rep_)) return false; + status_internal::StatusRep* rep = PrepareToModify(rep_); + auto res = rep->ErasePayload(type_url); + rep_ = res.new_rep; + return res.erased; } -inline uintptr_t Status::MovedFromRep() { - return CodeToInlinedRep(absl::StatusCode::kInternal) | 2; +inline void Status::ForEachPayload( + absl::FunctionRef<void(absl::string_view, const absl::Cord&)> visitor) + const { + if (IsInlined(rep_)) return; + RepToPointer(rep_)->ForEachPayload(visitor); } -inline uintptr_t Status::CodeToInlinedRep(absl::StatusCode code) { - return static_cast<uintptr_t>(code) << 2; +constexpr bool Status::IsInlined(uintptr_t rep) { return (rep & 1) != 0; } + +constexpr bool Status::IsMovedFrom(uintptr_t rep) { return (rep & 2) != 0; } + +constexpr uintptr_t Status::CodeToInlinedRep(absl::StatusCode code) { + return (static_cast<uintptr_t>(code) << 2) + 1; } -inline absl::StatusCode Status::InlinedRepToCode(uintptr_t rep) { - assert(IsInlined(rep)); +constexpr absl::StatusCode Status::InlinedRepToCode(uintptr_t rep) { + ABSL_ASSERT(IsInlined(rep)); return static_cast<absl::StatusCode>(rep >> 2); } -inline status_internal::StatusRep* Status::RepToPointer(uintptr_t rep) { +constexpr uintptr_t Status::MovedFromRep() { + return CodeToInlinedRep(absl::StatusCode::kInternal) | 2; +} + +inline absl::Nonnull<const status_internal::StatusRep*> Status::RepToPointer( + uintptr_t rep) { assert(!IsInlined(rep)); - return reinterpret_cast<status_internal::StatusRep*>(rep - 1); + return reinterpret_cast<const status_internal::StatusRep*>(rep); } -inline uintptr_t Status::PointerToRep(status_internal::StatusRep* rep) { - return reinterpret_cast<uintptr_t>(rep) + 1; +inline uintptr_t Status::PointerToRep( + absl::Nonnull<status_internal::StatusRep*> rep) { + return reinterpret_cast<uintptr_t>(rep); } inline void Status::Ref(uintptr_t rep) { - if (!IsInlined(rep)) { - RepToPointer(rep)->ref.fetch_add(1, std::memory_order_relaxed); - } + if (!IsInlined(rep)) RepToPointer(rep)->Ref(); } inline void Status::Unref(uintptr_t rep) { - if (!IsInlined(rep)) { - UnrefNonInlined(rep); - } + if (!IsInlined(rep)) RepToPointer(rep)->Unref(); } inline Status OkStatus() { return Status(); } @@ -892,7 +934,7 @@ inline Status CancelledError() { return Status(absl::StatusCode::kCancelled); } // If the status's message is empty, the empty string is returned. // // StatusMessageAsCStr exists for C support. Use `status.message()` in C++. -const char* StatusMessageAsCStr( +absl::Nonnull<const char*> StatusMessageAsCStr( const Status& status ABSL_ATTRIBUTE_LIFETIME_BOUND); ABSL_NAMESPACE_END diff --git a/contrib/restricted/abseil-cpp/absl/status/status_payload_printer.cc b/contrib/restricted/abseil-cpp/absl/status/status_payload_printer.cc index a47aea11c2..98401e9072 100644 --- a/contrib/restricted/abseil-cpp/absl/status/status_payload_printer.cc +++ b/contrib/restricted/abseil-cpp/absl/status/status_payload_printer.cc @@ -13,9 +13,7 @@ // limitations under the License. #include "absl/status/status_payload_printer.h" -#include <atomic> - -#include "absl/base/attributes.h" +#include "absl/base/config.h" #include "absl/base/internal/atomic_hook.h" namespace absl { diff --git a/contrib/restricted/abseil-cpp/absl/status/status_payload_printer.h b/contrib/restricted/abseil-cpp/absl/status/status_payload_printer.h index 5e0937f67d..f22255e1c6 100644 --- a/contrib/restricted/abseil-cpp/absl/status/status_payload_printer.h +++ b/contrib/restricted/abseil-cpp/absl/status/status_payload_printer.h @@ -16,6 +16,7 @@ #include <string> +#include "absl/base/nullability.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" @@ -34,8 +35,8 @@ namespace status_internal { // NOTE: This is an internal API and the design is subject to change in the // future in a non-backward-compatible way. Since it's only meant for debugging // purpose, you should not rely on it in any critical logic. -using StatusPayloadPrinter = absl::optional<std::string> (*)(absl::string_view, - const absl::Cord&); +using StatusPayloadPrinter = absl::Nullable<absl::optional<std::string> (*)( + absl::string_view, const absl::Cord&)>; // Sets the global payload printer. Only one printer should be set per process. // If multiple printers are set, it's undefined which one will be used. diff --git a/contrib/restricted/abseil-cpp/absl/status/statusor.cc b/contrib/restricted/abseil-cpp/absl/status/statusor.cc index 96642b340f..7e6b334c74 100644 --- a/contrib/restricted/abseil-cpp/absl/status/statusor.cc +++ b/contrib/restricted/abseil-cpp/absl/status/statusor.cc @@ -17,7 +17,10 @@ #include <utility> #include "absl/base/call_once.h" +#include "absl/base/config.h" #include "absl/base/internal/raw_logging.h" +#include "absl/base/nullability.h" +#include "absl/status/internal/statusor_internal.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" @@ -52,7 +55,7 @@ BadStatusOrAccess& BadStatusOrAccess::operator=(BadStatusOrAccess&& other) { BadStatusOrAccess::BadStatusOrAccess(BadStatusOrAccess&& other) : status_(std::move(other.status_)) {} -const char* BadStatusOrAccess::what() const noexcept { +absl::Nonnull<const char*> BadStatusOrAccess::what() const noexcept { InitWhat(); return what_.c_str(); } @@ -67,7 +70,7 @@ void BadStatusOrAccess::InitWhat() const { namespace internal_statusor { -void Helper::HandleInvalidStatusCtorArg(absl::Status* status) { +void Helper::HandleInvalidStatusCtorArg(absl::Nonnull<absl::Status*> status) { const char* kMessage = "An OK status is not a valid constructor argument to StatusOr<T>"; #ifdef NDEBUG diff --git a/contrib/restricted/abseil-cpp/absl/status/statusor.h b/contrib/restricted/abseil-cpp/absl/status/statusor.h index 54c7ce023f..cd35e5b436 100644 --- a/contrib/restricted/abseil-cpp/absl/status/statusor.h +++ b/contrib/restricted/abseil-cpp/absl/status/statusor.h @@ -39,15 +39,20 @@ #include <exception> #include <initializer_list> #include <new> +#include <ostream> #include <string> #include <type_traits> #include <utility> #include "absl/base/attributes.h" +#include "absl/base/nullability.h" #include "absl/base/call_once.h" #include "absl/meta/type_traits.h" #include "absl/status/internal/statusor_internal.h" #include "absl/status/status.h" +#include "absl/strings/has_absl_stringify.h" +#include "absl/strings/has_ostream_operator.h" +#include "absl/strings/str_format.h" #include "absl/types/variant.h" #include "absl/utility/utility.h" @@ -88,7 +93,7 @@ class BadStatusOrAccess : public std::exception { // // The pointer of this string is guaranteed to be valid until any non-const // function is invoked on the exception object. - const char* what() const noexcept override; + absl::Nonnull<const char*> what() const noexcept override; // BadStatusOrAccess::status() // @@ -650,6 +655,41 @@ bool operator!=(const StatusOr<T>& lhs, const StatusOr<T>& rhs) { return !(lhs == rhs); } +// Prints the `value` or the status in brackets to `os`. +// +// Requires `T` supports `operator<<`. Do not rely on the output format which +// may change without notice. +template <typename T, typename std::enable_if< + absl::HasOstreamOperator<T>::value, int>::type = 0> +std::ostream& operator<<(std::ostream& os, const StatusOr<T>& status_or) { + if (status_or.ok()) { + os << status_or.value(); + } else { + os << internal_statusor::StringifyRandom::OpenBrackets() + << status_or.status() + << internal_statusor::StringifyRandom::CloseBrackets(); + } + return os; +} + +// As above, but supports `StrCat`, `StrFormat`, etc. +// +// Requires `T` has `AbslStringify`. Do not rely on the output format which +// may change without notice. +template < + typename Sink, typename T, + typename std::enable_if<absl::HasAbslStringify<T>::value, int>::type = 0> +void AbslStringify(Sink& sink, const StatusOr<T>& status_or) { + if (status_or.ok()) { + absl::Format(&sink, "%v", status_or.value()); + } else { + absl::Format(&sink, "%s%v%s", + internal_statusor::StringifyRandom::OpenBrackets(), + status_or.status(), + internal_statusor::StringifyRandom::CloseBrackets()); + } +} + //------------------------------------------------------------------------------ // Implementation details for StatusOr<T> //------------------------------------------------------------------------------ @@ -750,13 +790,13 @@ T&& StatusOr<T>::operator*() && { } template <typename T> -const T* StatusOr<T>::operator->() const { +absl::Nonnull<const T*> StatusOr<T>::operator->() const { this->EnsureOk(); return &this->data_; } template <typename T> -T* StatusOr<T>::operator->() { +absl::Nonnull<T*> StatusOr<T>::operator->() { this->EnsureOk(); return &this->data_; } diff --git a/contrib/restricted/abseil-cpp/absl/status/ya.make b/contrib/restricted/abseil-cpp/absl/status/ya.make index f4a990c542..48ab349f07 100644 --- a/contrib/restricted/abseil-cpp/absl/status/ya.make +++ b/contrib/restricted/abseil-cpp/absl/status/ya.make @@ -26,6 +26,7 @@ NO_COMPILER_WARNINGS() NO_UTIL() SRCS( + internal/status_internal.cc status.cc status_payload_printer.cc ) diff --git a/contrib/restricted/abseil-cpp/absl/strings/ascii.cc b/contrib/restricted/abseil-cpp/absl/strings/ascii.cc index 16c9689961..5460b2c6dc 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/ascii.cc +++ b/contrib/restricted/abseil-cpp/absl/strings/ascii.cc @@ -15,8 +15,13 @@ #include "absl/strings/ascii.h" #include <climits> +#include <cstdint> #include <cstring> #include <string> +#include <type_traits> + +#include "absl/base/config.h" +#include "absl/base/nullability.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -157,8 +162,74 @@ ABSL_DLL const char kToUpper[256] = { }; // clang-format on +template <class T> +static constexpr T BroadcastByte(unsigned char value) { + static_assert(std::is_integral<T>::value && sizeof(T) <= sizeof(uint64_t) && + std::is_unsigned<T>::value, + "only unsigned integers up to 64-bit allowed"); + T result = value; + constexpr size_t result_bit_width = sizeof(result) * CHAR_BIT; + result |= result << ((CHAR_BIT << 0) & (result_bit_width - 1)); + result |= result << ((CHAR_BIT << 1) & (result_bit_width - 1)); + result |= result << ((CHAR_BIT << 2) & (result_bit_width - 1)); + return result; +} + +// Returns whether `c` is in the a-z/A-Z range (w.r.t. `ToUpper`). +// Implemented by: +// 1. Pushing the a-z/A-Z range to [SCHAR_MIN, SCHAR_MIN + 26). +// 2. Comparing to SCHAR_MIN + 26. +template <bool ToUpper> +constexpr bool AsciiInAZRange(unsigned char c) { + constexpr unsigned char sub = (ToUpper ? 'a' : 'A') - SCHAR_MIN; + constexpr signed char threshold = SCHAR_MIN + 26; // 26 = alphabet size. + // Using unsigned arithmetic as overflows/underflows are well defined. + unsigned char u = c - sub; + // Using signed cmp, as SIMD unsigned cmp isn't available in many platforms. + return static_cast<signed char>(u) < threshold; +} + template <bool ToUpper> -constexpr void AsciiStrCaseFold(char* p, char* end) { +static constexpr char* PartialAsciiStrCaseFold(absl::Nonnull<char*> p, + absl::Nonnull<char*> end) { + using vec_t = size_t; + const size_t n = static_cast<size_t>(end - p); + + // SWAR algorithm: http://0x80.pl/notesen/2016-01-06-swar-swap-case.html + constexpr char ch_a = ToUpper ? 'a' : 'A', ch_z = ToUpper ? 'z' : 'Z'; + char* const swar_end = p + (n / sizeof(vec_t)) * sizeof(vec_t); + while (p < swar_end) { + vec_t v = vec_t(); + + // memcpy the vector, but constexpr + for (size_t i = 0; i < sizeof(vec_t); ++i) { + v |= static_cast<vec_t>(static_cast<unsigned char>(p[i])) + << (i * CHAR_BIT); + } + + constexpr unsigned int msb = 1u << (CHAR_BIT - 1); + const vec_t v_msb = v & BroadcastByte<vec_t>(msb); + const vec_t v_nonascii_mask = (v_msb << 1) - (v_msb >> (CHAR_BIT - 1)); + const vec_t v_nonascii = v & v_nonascii_mask; + const vec_t v_ascii = v & ~v_nonascii_mask; + const vec_t a = v_ascii + BroadcastByte<vec_t>(msb - ch_a - 0), + z = v_ascii + BroadcastByte<vec_t>(msb - ch_z - 1); + v = v_nonascii | (v_ascii ^ ((a ^ z) & BroadcastByte<vec_t>(msb)) >> 2); + + // memcpy the vector, but constexpr + for (size_t i = 0; i < sizeof(vec_t); ++i) { + p[i] = static_cast<char>(v >> (i * CHAR_BIT)); + } + + p += sizeof(v); + } + + return p; +} + +template <bool ToUpper> +static constexpr void AsciiStrCaseFold(absl::Nonnull<char*> p, + absl::Nonnull<char*> end) { // The upper- and lowercase versions of ASCII characters differ by only 1 bit. // When we need to flip the case, we can xor with this bit to achieve the // desired result. Note that the choice of 'a' and 'A' here is arbitrary. We @@ -166,16 +237,17 @@ constexpr void AsciiStrCaseFold(char* p, char* end) { // have the same single bit difference. constexpr unsigned char kAsciiCaseBitFlip = 'a' ^ 'A'; - constexpr char ch_a = ToUpper ? 'a' : 'A'; - constexpr char ch_z = ToUpper ? 'z' : 'Z'; - for (; p < end; ++p) { + using vec_t = size_t; + // TODO(b/316380338): When FDO becomes able to vectorize these, + // revert this manual optimization and just leave the naive loop. + if (static_cast<size_t>(end - p) >= sizeof(vec_t)) { + p = ascii_internal::PartialAsciiStrCaseFold<ToUpper>(p, end); + } + while (p < end) { unsigned char v = static_cast<unsigned char>(*p); - // We use & instead of && to ensure this always stays branchless - // We use static_cast<int> to suppress -Wbitwise-instead-of-logical - bool is_in_range = static_cast<bool>(static_cast<int>(ch_a <= v) & - static_cast<int>(v <= ch_z)); - v ^= is_in_range ? kAsciiCaseBitFlip : 0; + v ^= AsciiInAZRange<ToUpper>(v) ? kAsciiCaseBitFlip : 0; *p = static_cast<char>(v); + ++p; } } @@ -205,17 +277,17 @@ static_assert(ValidateAsciiCasefold() == 0, "error in case conversion"); } // namespace ascii_internal -void AsciiStrToLower(std::string* s) { +void AsciiStrToLower(absl::Nonnull<std::string*> s) { char* p = &(*s)[0]; // Guaranteed to be valid for empty strings return ascii_internal::AsciiStrCaseFold<false>(p, p + s->size()); } -void AsciiStrToUpper(std::string* s) { +void AsciiStrToUpper(absl::Nonnull<std::string*> s) { char* p = &(*s)[0]; // Guaranteed to be valid for empty strings return ascii_internal::AsciiStrCaseFold<true>(p, p + s->size()); } -void RemoveExtraAsciiWhitespace(std::string* str) { +void RemoveExtraAsciiWhitespace(absl::Nonnull<std::string*> str) { auto stripped = StripAsciiWhitespace(*str); if (stripped.empty()) { diff --git a/contrib/restricted/abseil-cpp/absl/strings/ascii.h b/contrib/restricted/abseil-cpp/absl/strings/ascii.h index 42eadaea6c..c238f4de82 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/ascii.h +++ b/contrib/restricted/abseil-cpp/absl/strings/ascii.h @@ -53,10 +53,12 @@ #define ABSL_STRINGS_ASCII_H_ #include <algorithm> +#include <cstddef> #include <string> #include "absl/base/attributes.h" #include "absl/base/config.h" +#include "absl/base/nullability.h" #include "absl/strings/string_view.h" namespace absl { @@ -165,7 +167,7 @@ inline char ascii_tolower(unsigned char c) { } // Converts the characters in `s` to lowercase, changing the contents of `s`. -void AsciiStrToLower(std::string* s); +void AsciiStrToLower(absl::Nonnull<std::string*> s); // Creates a lowercase string from a given absl::string_view. ABSL_MUST_USE_RESULT inline std::string AsciiStrToLower(absl::string_view s) { @@ -183,7 +185,7 @@ inline char ascii_toupper(unsigned char c) { } // Converts the characters in `s` to uppercase, changing the contents of `s`. -void AsciiStrToUpper(std::string* s); +void AsciiStrToUpper(absl::Nonnull<std::string*> s); // Creates an uppercase string from a given absl::string_view. ABSL_MUST_USE_RESULT inline std::string AsciiStrToUpper(absl::string_view s) { @@ -201,7 +203,7 @@ ABSL_MUST_USE_RESULT inline absl::string_view StripLeadingAsciiWhitespace( } // Strips in place whitespace from the beginning of the given string. -inline void StripLeadingAsciiWhitespace(std::string* str) { +inline void StripLeadingAsciiWhitespace(absl::Nonnull<std::string*> str) { auto it = std::find_if_not(str->begin(), str->end(), absl::ascii_isspace); str->erase(str->begin(), it); } @@ -215,7 +217,7 @@ ABSL_MUST_USE_RESULT inline absl::string_view StripTrailingAsciiWhitespace( } // Strips in place whitespace from the end of the given string -inline void StripTrailingAsciiWhitespace(std::string* str) { +inline void StripTrailingAsciiWhitespace(absl::Nonnull<std::string*> str) { auto it = std::find_if_not(str->rbegin(), str->rend(), absl::ascii_isspace); str->erase(static_cast<size_t>(str->rend() - it)); } @@ -228,13 +230,13 @@ ABSL_MUST_USE_RESULT inline absl::string_view StripAsciiWhitespace( } // Strips in place whitespace from both ends of the given string -inline void StripAsciiWhitespace(std::string* str) { +inline void StripAsciiWhitespace(absl::Nonnull<std::string*> str) { StripTrailingAsciiWhitespace(str); StripLeadingAsciiWhitespace(str); } // Removes leading, trailing, and consecutive internal whitespace. -void RemoveExtraAsciiWhitespace(std::string*); +void RemoveExtraAsciiWhitespace(absl::Nonnull<std::string*> str); ABSL_NAMESPACE_END } // namespace absl diff --git a/contrib/restricted/abseil-cpp/absl/strings/charconv.cc b/contrib/restricted/abseil-cpp/absl/strings/charconv.cc index 778a1c75a5..0c9227f849 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/charconv.cc +++ b/contrib/restricted/abseil-cpp/absl/strings/charconv.cc @@ -16,12 +16,14 @@ #include <algorithm> #include <cassert> -#include <cmath> -#include <cstring> +#include <cstddef> +#include <cstdint> #include <limits> +#include <system_error> // NOLINT(build/c++11) #include "absl/base/casts.h" #include "absl/base/config.h" +#include "absl/base/nullability.h" #include "absl/numeric/bits.h" #include "absl/numeric/int128.h" #include "absl/strings/internal/charconv_bigint.h" @@ -118,7 +120,7 @@ struct FloatTraits<double> { // Parsing a smaller N will produce something finite. static constexpr int kEiselLemireMaxExclusiveExp10 = 309; - static double MakeNan(const char* tagp) { + static double MakeNan(absl::Nonnull<const char*> tagp) { #if ABSL_HAVE_BUILTIN(__builtin_nan) // Use __builtin_nan() if available since it has a fix for // https://bugs.llvm.org/show_bug.cgi?id=37778 @@ -191,7 +193,7 @@ struct FloatTraits<float> { static constexpr int kEiselLemireMinInclusiveExp10 = -46 - 18; static constexpr int kEiselLemireMaxExclusiveExp10 = 39; - static float MakeNan(const char* tagp) { + static float MakeNan(absl::Nonnull<const char*> tagp) { #if ABSL_HAVE_BUILTIN(__builtin_nanf) // Use __builtin_nanf() if available since it has a fix for // https://bugs.llvm.org/show_bug.cgi?id=37778 @@ -343,7 +345,7 @@ int NormalizedShiftSize(int mantissa_width, int binary_exponent) { // `value` must be wider than the requested bit width. // // Returns the number of bits shifted. -int TruncateToBitWidth(int bit_width, uint128* value) { +int TruncateToBitWidth(int bit_width, absl::Nonnull<uint128*> value) { const int current_bit_width = BitWidth(*value); const int shift = current_bit_width - bit_width; *value >>= shift; @@ -355,7 +357,7 @@ int TruncateToBitWidth(int bit_width, uint128* value) { // the appropriate double, and returns true. template <typename FloatType> bool HandleEdgeCase(const strings_internal::ParsedFloat& input, bool negative, - FloatType* value) { + absl::Nonnull<FloatType*> value) { if (input.type == strings_internal::FloatType::kNan) { // A bug in both clang < 7 and gcc would cause the compiler to optimize // away the buffer we are building below. Declaring the buffer volatile @@ -404,7 +406,8 @@ bool HandleEdgeCase(const strings_internal::ParsedFloat& input, bool negative, // number is stored in *value. template <typename FloatType> void EncodeResult(const CalculatedFloat& calculated, bool negative, - absl::from_chars_result* result, FloatType* value) { + absl::Nonnull<absl::from_chars_result*> result, + absl::Nonnull<FloatType*> value) { if (calculated.exponent == kOverflow) { result->ec = std::errc::result_out_of_range; *value = negative ? -std::numeric_limits<FloatType>::max() @@ -450,7 +453,7 @@ void EncodeResult(const CalculatedFloat& calculated, bool negative, // Zero and negative values of `shift` are accepted, in which case the word is // shifted left, as necessary. uint64_t ShiftRightAndRound(uint128 value, int shift, bool input_exact, - bool* output_exact) { + absl::Nonnull<bool*> output_exact) { if (shift <= 0) { *output_exact = input_exact; return static_cast<uint64_t>(value << -shift); @@ -684,7 +687,8 @@ CalculatedFloat CalculateFromParsedDecimal( // this function returns false) is both fast and correct. template <typename FloatType> bool EiselLemire(const strings_internal::ParsedFloat& input, bool negative, - FloatType* value, std::errc* ec) { + absl::Nonnull<FloatType*> value, + absl::Nonnull<std::errc*> ec) { uint64_t man = input.mantissa; int exp10 = input.exponent; if (exp10 < FloatTraits<FloatType>::kEiselLemireMinInclusiveExp10) { @@ -857,7 +861,8 @@ bool EiselLemire(const strings_internal::ParsedFloat& input, bool negative, } template <typename FloatType> -from_chars_result FromCharsImpl(const char* first, const char* last, +from_chars_result FromCharsImpl(absl::Nonnull<const char*> first, + absl::Nonnull<const char*> last, FloatType& value, chars_format fmt_flags) { from_chars_result result; result.ptr = first; // overwritten on successful parse @@ -943,12 +948,14 @@ from_chars_result FromCharsImpl(const char* first, const char* last, } } // namespace -from_chars_result from_chars(const char* first, const char* last, double& value, +from_chars_result from_chars(absl::Nonnull<const char*> first, + absl::Nonnull<const char*> last, double& value, chars_format fmt) { return FromCharsImpl(first, last, value, fmt); } -from_chars_result from_chars(const char* first, const char* last, float& value, +from_chars_result from_chars(absl::Nonnull<const char*> first, + absl::Nonnull<const char*> last, float& value, chars_format fmt) { return FromCharsImpl(first, last, value, fmt); } diff --git a/contrib/restricted/abseil-cpp/absl/strings/charconv.h b/contrib/restricted/abseil-cpp/absl/strings/charconv.h index 111c7120de..be250902ad 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/charconv.h +++ b/contrib/restricted/abseil-cpp/absl/strings/charconv.h @@ -18,6 +18,7 @@ #include <system_error> // NOLINT(build/c++11) #include "absl/base/config.h" +#include "absl/base/nullability.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -44,7 +45,7 @@ enum class chars_format { // characters that were successfully parsed. If none was found, `ptr` is set // to the `first` argument to from_chars. struct from_chars_result { - const char* ptr; + absl::Nonnull<const char*> ptr; std::errc ec; }; @@ -76,11 +77,13 @@ struct from_chars_result { // format that strtod() accepts, except that a "0x" prefix is NOT matched. // (In particular, in `hex` mode, the input "0xff" results in the largest // matching pattern "0".) -absl::from_chars_result from_chars(const char* first, const char* last, +absl::from_chars_result from_chars(absl::Nonnull<const char*> first, + absl::Nonnull<const char*> last, double& value, // NOLINT chars_format fmt = chars_format::general); -absl::from_chars_result from_chars(const char* first, const char* last, +absl::from_chars_result from_chars(absl::Nonnull<const char*> first, + absl::Nonnull<const char*> last, float& value, // NOLINT chars_format fmt = chars_format::general); diff --git a/contrib/restricted/abseil-cpp/absl/strings/charset.h b/contrib/restricted/abseil-cpp/absl/strings/charset.h new file mode 100644 index 0000000000..ff4e81a41f --- /dev/null +++ b/contrib/restricted/abseil-cpp/absl/strings/charset.h @@ -0,0 +1,164 @@ +// Copyright 2022 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: charset.h +// ----------------------------------------------------------------------------- +// +// This file contains absl::CharSet, a fast, bit-vector set of 8-bit unsigned +// characters. +// +// Instances can be initialized as constexpr constants. For example: +// +// constexpr absl::CharSet kJustX = absl::CharSet::Char('x'); +// constexpr absl::CharSet kMySymbols = absl::CharSet("$@!"); +// constexpr absl::CharSet kLetters = absl::CharSet::Range('a', 'z'); +// +// Multiple instances can be combined that still forms a constexpr expression. +// For example: +// +// constexpr absl::CharSet kLettersAndNumbers = +// absl::CharSet::Range('a', 'z') | absl::CharSet::Range('0', '9'); +// +// Several pre-defined character classes are available that mirror the methods +// from <cctype>. For example: +// +// constexpr absl::CharSet kLettersAndWhitespace = +// absl::CharSet::AsciiAlphabet() | absl::CharSet::AsciiWhitespace(); +// +// To check membership, use the .contains method, e.g. +// +// absl::CharSet hex_letters("abcdef"); +// hex_letters.contains('a'); // true +// hex_letters.contains('g'); // false + +#ifndef ABSL_STRINGS_CHARSET_H_ +#define ABSL_STRINGS_CHARSET_H_ + +#include <cstddef> +#include <cstdint> +#include <cstring> + +#include "absl/base/macros.h" +#include "absl/base/port.h" +#include "absl/strings/string_view.h" + +namespace absl { + +class CharSet { + public: + constexpr CharSet() : m_() {} + + // Initializes with a given string_view. + constexpr explicit CharSet(absl::string_view str) : m_() { + for (char c : str) { + SetChar(static_cast<unsigned char>(c)); + } + } + + constexpr bool contains(char c) const { + return ((m_[static_cast<unsigned char>(c) / 64] >> + (static_cast<unsigned char>(c) % 64)) & + 0x1) == 0x1; + } + + constexpr bool empty() const { + for (uint64_t c : m_) { + if (c != 0) return false; + } + return true; + } + + // Containing only a single specified char. + static constexpr CharSet Char(char x) { + return CharSet(CharMaskForWord(x, 0), CharMaskForWord(x, 1), + CharMaskForWord(x, 2), CharMaskForWord(x, 3)); + } + + // Containing all the chars in the closed interval [lo,hi]. + static constexpr CharSet Range(char lo, char hi) { + return CharSet(RangeForWord(lo, hi, 0), RangeForWord(lo, hi, 1), + RangeForWord(lo, hi, 2), RangeForWord(lo, hi, 3)); + } + + friend constexpr CharSet operator&(const CharSet& a, const CharSet& b) { + return CharSet(a.m_[0] & b.m_[0], a.m_[1] & b.m_[1], a.m_[2] & b.m_[2], + a.m_[3] & b.m_[3]); + } + + friend constexpr CharSet operator|(const CharSet& a, const CharSet& b) { + return CharSet(a.m_[0] | b.m_[0], a.m_[1] | b.m_[1], a.m_[2] | b.m_[2], + a.m_[3] | b.m_[3]); + } + + friend constexpr CharSet operator~(const CharSet& a) { + return CharSet(~a.m_[0], ~a.m_[1], ~a.m_[2], ~a.m_[3]); + } + + // Mirrors the char-classifying predicates in <cctype>. + static constexpr CharSet AsciiUppercase() { return CharSet::Range('A', 'Z'); } + static constexpr CharSet AsciiLowercase() { return CharSet::Range('a', 'z'); } + static constexpr CharSet AsciiDigits() { return CharSet::Range('0', '9'); } + static constexpr CharSet AsciiAlphabet() { + return AsciiLowercase() | AsciiUppercase(); + } + static constexpr CharSet AsciiAlphanumerics() { + return AsciiDigits() | AsciiAlphabet(); + } + static constexpr CharSet AsciiHexDigits() { + return AsciiDigits() | CharSet::Range('A', 'F') | CharSet::Range('a', 'f'); + } + static constexpr CharSet AsciiPrintable() { + return CharSet::Range(0x20, 0x7e); + } + static constexpr CharSet AsciiWhitespace() { return CharSet("\t\n\v\f\r "); } + static constexpr CharSet AsciiPunctuation() { + return AsciiPrintable() & ~AsciiWhitespace() & ~AsciiAlphanumerics(); + } + + private: + constexpr CharSet(uint64_t b0, uint64_t b1, uint64_t b2, uint64_t b3) + : m_{b0, b1, b2, b3} {} + + static constexpr uint64_t RangeForWord(char lo, char hi, uint64_t word) { + return OpenRangeFromZeroForWord(static_cast<unsigned char>(hi) + 1, word) & + ~OpenRangeFromZeroForWord(static_cast<unsigned char>(lo), word); + } + + // All the chars in the specified word of the range [0, upper). + static constexpr uint64_t OpenRangeFromZeroForWord(uint64_t upper, + uint64_t word) { + return (upper <= 64 * word) ? 0 + : (upper >= 64 * (word + 1)) + ? ~static_cast<uint64_t>(0) + : (~static_cast<uint64_t>(0) >> (64 - upper % 64)); + } + + static constexpr uint64_t CharMaskForWord(char x, uint64_t word) { + return (static_cast<unsigned char>(x) / 64 == word) + ? (static_cast<uint64_t>(1) + << (static_cast<unsigned char>(x) % 64)) + : 0; + } + + constexpr void SetChar(unsigned char c) { + m_[c / 64] |= static_cast<uint64_t>(1) << (c % 64); + } + + uint64_t m_[4]; +}; + +} // namespace absl + +#endif // ABSL_STRINGS_CHARSET_H_ diff --git a/contrib/restricted/abseil-cpp/absl/strings/cord.cc b/contrib/restricted/abseil-cpp/absl/strings/cord.cc index 14976aef58..f67326fdef 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/cord.cc +++ b/contrib/restricted/abseil-cpp/absl/strings/cord.cc @@ -15,27 +15,33 @@ #include "absl/strings/cord.h" #include <algorithm> -#include <atomic> +#include <cassert> #include <cstddef> +#include <cstdint> #include <cstdio> #include <cstdlib> +#include <cstring> #include <iomanip> #include <ios> #include <iostream> #include <limits> +#include <memory> #include <ostream> #include <sstream> -#include <type_traits> -#include <unordered_set> -#include <vector> +#include <string> +#include <utility> -#include "absl/base/casts.h" +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/internal/endian.h" #include "absl/base/internal/raw_logging.h" #include "absl/base/macros.h" -#include "absl/base/port.h" -#include "absl/container/fixed_array.h" +#include "absl/base/optimization.h" +#include "absl/base/nullability.h" #include "absl/container/inlined_vector.h" +#include "absl/crc/crc32c.h" #include "absl/crc/internal/crc_cord_state.h" +#include "absl/functional/function_ref.h" #include "absl/strings/cord_buffer.h" #include "absl/strings/escaping.h" #include "absl/strings/internal/cord_data_edge.h" @@ -43,13 +49,14 @@ #include "absl/strings/internal/cord_rep_btree.h" #include "absl/strings/internal/cord_rep_crc.h" #include "absl/strings/internal/cord_rep_flat.h" -#include "absl/strings/internal/cordz_statistics.h" -#include "absl/strings/internal/cordz_update_scope.h" #include "absl/strings/internal/cordz_update_tracker.h" #include "absl/strings/internal/resize_uninitialized.h" +#include "absl/strings/match.h" #include "absl/strings/str_cat.h" -#include "absl/strings/str_join.h" #include "absl/strings/string_view.h" +#include "absl/strings/strip.h" +#include "absl/types/optional.h" +#include "absl/types/span.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -68,29 +75,21 @@ using ::absl::cord_internal::kMinFlatLength; using ::absl::cord_internal::kInlinedVectorSize; using ::absl::cord_internal::kMaxBytesToCopy; -static void DumpNode(CordRep* rep, bool include_data, std::ostream* os, - int indent = 0); -static bool VerifyNode(CordRep* root, CordRep* start_node, - bool full_validation); - -static inline CordRep* VerifyTree(CordRep* node) { - // Verification is expensive, so only do it in debug mode. - // Even in debug mode we normally do only light validation. - // If you are debugging Cord itself, you should define the - // macro EXTRA_CORD_VALIDATION, e.g. by adding - // --copt=-DEXTRA_CORD_VALIDATION to the blaze line. -#ifdef EXTRA_CORD_VALIDATION - assert(node == nullptr || VerifyNode(node, node, /*full_validation=*/true)); -#else // EXTRA_CORD_VALIDATION - assert(node == nullptr || VerifyNode(node, node, /*full_validation=*/false)); -#endif // EXTRA_CORD_VALIDATION - static_cast<void>(&VerifyNode); +static void DumpNode(absl::Nonnull<CordRep*> rep, bool include_data, + absl::Nonnull<std::ostream*> os, int indent = 0); +static bool VerifyNode(absl::Nonnull<CordRep*> root, + absl::Nonnull<CordRep*> start_node); +static inline absl::Nullable<CordRep*> VerifyTree( + absl::Nullable<CordRep*> node) { + assert(node == nullptr || VerifyNode(node, node)); + static_cast<void>(&VerifyNode); return node; } -static CordRepFlat* CreateFlat(const char* data, size_t length, - size_t alloc_hint) { +static absl::Nonnull<CordRepFlat*> CreateFlat(absl::Nonnull<const char*> data, + size_t length, + size_t alloc_hint) { CordRepFlat* flat = CordRepFlat::New(length + alloc_hint); flat->length = length; memcpy(flat->Data(), data, length); @@ -99,7 +98,8 @@ static CordRepFlat* CreateFlat(const char* data, size_t length, // Creates a new flat or Btree out of the specified array. // The returned node has a refcount of 1. -static CordRep* NewBtree(const char* data, size_t length, size_t alloc_hint) { +static absl::Nonnull<CordRep*> NewBtree(absl::Nonnull<const char*> data, + size_t length, size_t alloc_hint) { if (length <= kMaxFlatLength) { return CreateFlat(data, length, alloc_hint); } @@ -112,14 +112,16 @@ static CordRep* NewBtree(const char* data, size_t length, size_t alloc_hint) { // Create a new tree out of the specified array. // The returned node has a refcount of 1. -static CordRep* NewTree(const char* data, size_t length, size_t alloc_hint) { +static absl::Nullable<CordRep*> NewTree(absl::Nullable<const char*> data, + size_t length, size_t alloc_hint) { if (length == 0) return nullptr; return NewBtree(data, length, alloc_hint); } namespace cord_internal { -void InitializeCordRepExternal(absl::string_view data, CordRepExternal* rep) { +void InitializeCordRepExternal(absl::string_view data, + absl::Nonnull<CordRepExternal*> rep) { assert(!data.empty()); rep->length = data.size(); rep->tag = EXTERNAL; @@ -133,7 +135,7 @@ void InitializeCordRepExternal(absl::string_view data, CordRepExternal* rep) { // and not wasteful, we move the string into an external cord rep, preserving // the already allocated string contents. // Requires the provided string length to be larger than `kMaxInline`. -static CordRep* CordRepFromString(std::string&& src) { +static absl::Nonnull<CordRep*> CordRepFromString(std::string&& src) { assert(src.length() > cord_internal::kMaxInline); if ( // String is short: copy data to avoid external block overhead. @@ -165,12 +167,13 @@ static CordRep* CordRepFromString(std::string&& src) { constexpr unsigned char Cord::InlineRep::kMaxInline; #endif -inline void Cord::InlineRep::set_data(const char* data, size_t n) { +inline void Cord::InlineRep::set_data(absl::Nonnull<const char*> data, + size_t n) { static_assert(kMaxInline == 15, "set_data is hard-coded for a length of 15"); data_.set_inline_data(data, n); } -inline char* Cord::InlineRep::set_data(size_t n) { +inline absl::Nonnull<char*> Cord::InlineRep::set_data(size_t n) { assert(n <= kMaxInline); ResetToEmpty(); set_inline_size(n); @@ -194,13 +197,13 @@ inline void Cord::InlineRep::remove_prefix(size_t n) { // Returns `rep` converted into a CordRepBtree. // Directly returns `rep` if `rep` is already a CordRepBtree. -static CordRepBtree* ForceBtree(CordRep* rep) { +static absl::Nonnull<CordRepBtree*> ForceBtree(CordRep* rep) { return rep->IsBtree() ? rep->btree() : CordRepBtree::Create(cord_internal::RemoveCrcNode(rep)); } -void Cord::InlineRep::AppendTreeToInlined(CordRep* tree, +void Cord::InlineRep::AppendTreeToInlined(absl::Nonnull<CordRep*> tree, MethodIdentifier method) { assert(!is_tree()); if (!data_.is_empty()) { @@ -210,14 +213,16 @@ void Cord::InlineRep::AppendTreeToInlined(CordRep* tree, EmplaceTree(tree, method); } -void Cord::InlineRep::AppendTreeToTree(CordRep* tree, MethodIdentifier method) { +void Cord::InlineRep::AppendTreeToTree(absl::Nonnull<CordRep*> tree, + MethodIdentifier method) { assert(is_tree()); const CordzUpdateScope scope(data_.cordz_info(), method); tree = CordRepBtree::Append(ForceBtree(data_.as_tree()), tree); SetTree(tree, scope); } -void Cord::InlineRep::AppendTree(CordRep* tree, MethodIdentifier method) { +void Cord::InlineRep::AppendTree(absl::Nonnull<CordRep*> tree, + MethodIdentifier method) { assert(tree != nullptr); assert(tree->length != 0); assert(!tree->IsCrc()); @@ -228,7 +233,7 @@ void Cord::InlineRep::AppendTree(CordRep* tree, MethodIdentifier method) { } } -void Cord::InlineRep::PrependTreeToInlined(CordRep* tree, +void Cord::InlineRep::PrependTreeToInlined(absl::Nonnull<CordRep*> tree, MethodIdentifier method) { assert(!is_tree()); if (!data_.is_empty()) { @@ -238,7 +243,7 @@ void Cord::InlineRep::PrependTreeToInlined(CordRep* tree, EmplaceTree(tree, method); } -void Cord::InlineRep::PrependTreeToTree(CordRep* tree, +void Cord::InlineRep::PrependTreeToTree(absl::Nonnull<CordRep*> tree, MethodIdentifier method) { assert(is_tree()); const CordzUpdateScope scope(data_.cordz_info(), method); @@ -246,7 +251,8 @@ void Cord::InlineRep::PrependTreeToTree(CordRep* tree, SetTree(tree, scope); } -void Cord::InlineRep::PrependTree(CordRep* tree, MethodIdentifier method) { +void Cord::InlineRep::PrependTree(absl::Nonnull<CordRep*> tree, + MethodIdentifier method) { assert(tree != nullptr); assert(tree->length != 0); assert(!tree->IsCrc()); @@ -261,8 +267,9 @@ void Cord::InlineRep::PrependTree(CordRep* tree, MethodIdentifier method) { // suitable leaf is found, the function will update the length field for all // nodes to account for the size increase. The append region address will be // written to region and the actual size increase will be written to size. -static inline bool PrepareAppendRegion(CordRep* root, char** region, - size_t* size, size_t max_length) { +static inline bool PrepareAppendRegion( + absl::Nonnull<CordRep*> root, absl::Nonnull<absl::Nullable<char*>*> region, + absl::Nonnull<size_t*> size, size_t max_length) { if (root->IsBtree() && root->refcount.IsOne()) { Span<char> span = root->btree()->GetAppendBuffer(max_length); if (!span.empty()) { @@ -465,11 +472,11 @@ void Cord::InlineRep::AppendArray(absl::string_view src, CommitTree(root, rep, scope, method); } -inline CordRep* Cord::TakeRep() const& { +inline absl::Nonnull<CordRep*> Cord::TakeRep() const& { return CordRep::Ref(contents_.tree()); } -inline CordRep* Cord::TakeRep() && { +inline absl::Nonnull<CordRep*> Cord::TakeRep() && { CordRep* rep = contents_.tree(); contents_.clear(); return rep; @@ -527,7 +534,7 @@ inline void Cord::AppendImpl(C&& src) { contents_.AppendTree(rep, CordzUpdateTracker::kAppendCord); } -static CordRep::ExtractResult ExtractAppendBuffer(CordRep* rep, +static CordRep::ExtractResult ExtractAppendBuffer(absl::Nonnull<CordRep*> rep, size_t min_capacity) { switch (rep->tag) { case cord_internal::BTREE: @@ -573,13 +580,9 @@ CordBuffer Cord::GetAppendBufferSlowPath(size_t block_size, size_t capacity, return CreateAppendBuffer(contents_.data_, block_size, capacity); } -void Cord::Append(const Cord& src) { - AppendImpl(src); -} +void Cord::Append(const Cord& src) { AppendImpl(src); } -void Cord::Append(Cord&& src) { - AppendImpl(std::move(src)); -} +void Cord::Append(Cord&& src) { AppendImpl(std::move(src)); } template <typename T, Cord::EnableIfString<T>> void Cord::Append(T&& src) { @@ -778,8 +781,9 @@ int ClampResult(int memcmp_res) { return static_cast<int>(memcmp_res > 0) - static_cast<int>(memcmp_res < 0); } -int CompareChunks(absl::string_view* lhs, absl::string_view* rhs, - size_t* size_to_compare) { +int CompareChunks(absl::Nonnull<absl::string_view*> lhs, + absl::Nonnull<absl::string_view*> rhs, + absl::Nonnull<size_t*> size_to_compare) { size_t compared_size = std::min(lhs->size(), rhs->size()); assert(*size_to_compare >= compared_size); *size_to_compare -= compared_size; @@ -877,7 +881,8 @@ void Cord::SetExpectedChecksum(uint32_t crc) { SetCrcCordState(std::move(state)); } -const crc_internal::CrcCordState* Cord::MaybeGetCrcCordState() const { +absl::Nullable<const crc_internal::CrcCordState*> Cord::MaybeGetCrcCordState() + const { if (!contents_.is_tree() || !contents_.tree()->IsCrc()) { return nullptr; } @@ -894,7 +899,8 @@ absl::optional<uint32_t> Cord::ExpectedChecksum() const { inline int Cord::CompareSlowPath(absl::string_view rhs, size_t compared_size, size_t size_to_compare) const { - auto advance = [](Cord::ChunkIterator* it, absl::string_view* chunk) { + auto advance = [](absl::Nonnull<Cord::ChunkIterator*> it, + absl::Nonnull<absl::string_view*> chunk) { if (!chunk->empty()) return true; ++*it; if (it->bytes_remaining_ == 0) return false; @@ -924,7 +930,8 @@ inline int Cord::CompareSlowPath(absl::string_view rhs, size_t compared_size, inline int Cord::CompareSlowPath(const Cord& rhs, size_t compared_size, size_t size_to_compare) const { - auto advance = [](Cord::ChunkIterator* it, absl::string_view* chunk) { + auto advance = [](absl::Nonnull<Cord::ChunkIterator*> it, + absl::Nonnull<absl::string_view*> chunk) { if (!chunk->empty()) return true; ++*it; if (it->bytes_remaining_ == 0) return false; @@ -1046,7 +1053,7 @@ Cord::operator std::string() const { return s; } -void CopyCordToString(const Cord& src, std::string* dst) { +void CopyCordToString(const Cord& src, absl::Nonnull<std::string*> dst) { if (!src.contents_.is_tree()) { src.contents_.CopyTo(dst); } else { @@ -1055,7 +1062,7 @@ void CopyCordToString(const Cord& src, std::string* dst) { } } -void Cord::CopyToArraySlowPath(char* dst) const { +void Cord::CopyToArraySlowPath(absl::Nonnull<char*> dst) const { assert(contents_.is_tree()); absl::string_view fragment; if (GetFlatAux(contents_.tree(), &fragment)) { @@ -1165,6 +1172,194 @@ char Cord::operator[](size_t i) const { } } +namespace { + +// Tests whether the sequence of chunks beginning at `position` starts with +// `needle`. +// +// REQUIRES: remaining `absl::Cord` starting at `position` is greater than or +// equal to `needle.size()`. +bool IsSubstringInCordAt(absl::Cord::CharIterator position, + absl::string_view needle) { + auto haystack_chunk = absl::Cord::ChunkRemaining(position); + while (true) { + // Precondition is that `absl::Cord::ChunkRemaining(position)` is not + // empty. This assert will trigger if that is not true. + assert(!haystack_chunk.empty()); + auto min_length = std::min(haystack_chunk.size(), needle.size()); + if (!absl::ConsumePrefix(&needle, haystack_chunk.substr(0, min_length))) { + return false; + } + if (needle.empty()) { + return true; + } + absl::Cord::Advance(&position, min_length); + haystack_chunk = absl::Cord::ChunkRemaining(position); + } +} + +} // namespace + +// A few options how this could be implemented: +// (a) Flatten the Cord and find, i.e. +// haystack.Flatten().find(needle) +// For large 'haystack' (where Cord makes sense to be used), this copies +// the whole 'haystack' and can be slow. +// (b) Use std::search, i.e. +// std::search(haystack.char_begin(), haystack.char_end(), +// needle.begin(), needle.end()) +// This avoids the copy, but compares one byte at a time, and branches a +// lot every time it has to advance. It is also not possible to use +// std::search as is, because CharIterator is only an input iterator, not a +// forward iterator. +// (c) Use string_view::find in each fragment, and specifically handle fragment +// boundaries. +// +// This currently implements option (b). +absl::Cord::CharIterator absl::Cord::FindImpl(CharIterator it, + absl::string_view needle) const { + // Ensure preconditions are met by callers first. + + // Needle must not be empty. + assert(!needle.empty()); + // Haystack must be at least as large as needle. + assert(it.chunk_iterator_.bytes_remaining_ >= needle.size()); + + // Cord is a sequence of chunks. To find `needle` we go chunk by chunk looking + // for the first char of needle, up until we have advanced `N` defined as + // `haystack.size() - needle.size()`. If we find the first char of needle at + // `P` and `P` is less than `N`, we then call `IsSubstringInCordAt` to + // see if this is the needle. If not, we advance to `P + 1` and try again. + while (it.chunk_iterator_.bytes_remaining_ >= needle.size()) { + auto haystack_chunk = Cord::ChunkRemaining(it); + assert(!haystack_chunk.empty()); + // Look for the first char of `needle` in the current chunk. + auto idx = haystack_chunk.find(needle.front()); + if (idx == absl::string_view::npos) { + // No potential match in this chunk, advance past it. + Cord::Advance(&it, haystack_chunk.size()); + continue; + } + // We found the start of a potential match in the chunk. Advance the + // iterator and haystack chunk to the match the position. + Cord::Advance(&it, idx); + // Check if there is enough haystack remaining to actually have a match. + if (it.chunk_iterator_.bytes_remaining_ < needle.size()) { + break; + } + // Check if this is `needle`. + if (IsSubstringInCordAt(it, needle)) { + return it; + } + // No match, increment the iterator for the next attempt. + Cord::Advance(&it, 1); + } + // If we got here, we did not find `needle`. + return char_end(); +} + +absl::Cord::CharIterator absl::Cord::Find(absl::string_view needle) const { + if (needle.empty()) { + return char_begin(); + } + if (needle.size() > size()) { + return char_end(); + } + if (needle.size() == size()) { + return *this == needle ? char_begin() : char_end(); + } + return FindImpl(char_begin(), needle); +} + +namespace { + +// Tests whether the sequence of chunks beginning at `haystack` starts with the +// sequence of chunks beginning at `needle_begin` and extending to `needle_end`. +// +// REQUIRES: remaining `absl::Cord` starting at `position` is greater than or +// equal to `needle_end - needle_begin` and `advance`. +bool IsSubcordInCordAt(absl::Cord::CharIterator haystack, + absl::Cord::CharIterator needle_begin, + absl::Cord::CharIterator needle_end) { + while (needle_begin != needle_end) { + auto haystack_chunk = absl::Cord::ChunkRemaining(haystack); + assert(!haystack_chunk.empty()); + auto needle_chunk = absl::Cord::ChunkRemaining(needle_begin); + auto min_length = std::min(haystack_chunk.size(), needle_chunk.size()); + if (haystack_chunk.substr(0, min_length) != + needle_chunk.substr(0, min_length)) { + return false; + } + absl::Cord::Advance(&haystack, min_length); + absl::Cord::Advance(&needle_begin, min_length); + } + return true; +} + +// Tests whether the sequence of chunks beginning at `position` starts with the +// cord `needle`. +// +// REQUIRES: remaining `absl::Cord` starting at `position` is greater than or +// equal to `needle.size()`. +bool IsSubcordInCordAt(absl::Cord::CharIterator position, + const absl::Cord& needle) { + return IsSubcordInCordAt(position, needle.char_begin(), needle.char_end()); +} + +} // namespace + +absl::Cord::CharIterator absl::Cord::Find(const absl::Cord& needle) const { + if (needle.empty()) { + return char_begin(); + } + const auto needle_size = needle.size(); + if (needle_size > size()) { + return char_end(); + } + if (needle_size == size()) { + return *this == needle ? char_begin() : char_end(); + } + const auto needle_chunk = Cord::ChunkRemaining(needle.char_begin()); + auto haystack_it = char_begin(); + while (true) { + haystack_it = FindImpl(haystack_it, needle_chunk); + if (haystack_it == char_end() || + haystack_it.chunk_iterator_.bytes_remaining_ < needle_size) { + break; + } + // We found the first chunk of `needle` at `haystack_it` but not the entire + // subcord. Advance past the first chunk and check for the remainder. + auto haystack_advanced_it = haystack_it; + auto needle_it = needle.char_begin(); + Cord::Advance(&haystack_advanced_it, needle_chunk.size()); + Cord::Advance(&needle_it, needle_chunk.size()); + if (IsSubcordInCordAt(haystack_advanced_it, needle_it, needle.char_end())) { + return haystack_it; + } + Cord::Advance(&haystack_it, 1); + if (haystack_it.chunk_iterator_.bytes_remaining_ < needle_size) { + break; + } + if (haystack_it.chunk_iterator_.bytes_remaining_ == needle_size) { + // Special case, if there is exactly `needle_size` bytes remaining, the + // subcord is either at `haystack_it` or not at all. + if (IsSubcordInCordAt(haystack_it, needle)) { + return haystack_it; + } + break; + } + } + return char_end(); +} + +bool Cord::Contains(absl::string_view rhs) const { + return rhs.empty() || Find(rhs) != char_end(); +} + +bool Cord::Contains(const absl::Cord& rhs) const { + return rhs.empty() || Find(rhs) != char_end(); +} + absl::string_view Cord::FlattenSlowPath() { assert(contents_.is_tree()); size_t total_size = size(); @@ -1193,7 +1388,8 @@ absl::string_view Cord::FlattenSlowPath() { return absl::string_view(new_buffer, total_size); } -/* static */ bool Cord::GetFlatAux(CordRep* rep, absl::string_view* fragment) { +/* static */ bool Cord::GetFlatAux(absl::Nonnull<CordRep*> rep, + absl::Nonnull<absl::string_view*> fragment) { assert(rep != nullptr); if (rep->length == 0) { *fragment = absl::string_view(); @@ -1227,7 +1423,7 @@ absl::string_view Cord::FlattenSlowPath() { } /* static */ void Cord::ForEachChunkAux( - absl::cord_internal::CordRep* rep, + absl::Nonnull<absl::cord_internal::CordRep*> rep, absl::FunctionRef<void(absl::string_view)> callback) { assert(rep != nullptr); if (rep->length == 0) return; @@ -1252,8 +1448,8 @@ absl::string_view Cord::FlattenSlowPath() { } } -static void DumpNode(CordRep* rep, bool include_data, std::ostream* os, - int indent) { +static void DumpNode(absl::Nonnull<CordRep*> rep, bool include_data, + absl::Nonnull<std::ostream*> os, int indent) { const int kIndentStep = 1; absl::InlinedVector<CordRep*, kInlinedVectorSize> stack; absl::InlinedVector<int, kInlinedVectorSize> indents; @@ -1289,7 +1485,7 @@ static void DumpNode(CordRep* rep, bool include_data, std::ostream* os, *os << absl::CEscape(std::string(rep->flat()->Data(), rep->length)); *os << "]\n"; } else { - CordRepBtree::Dump(rep, /*label=*/ "", include_data, *os); + CordRepBtree::Dump(rep, /*label=*/"", include_data, *os); } } if (leaf) { @@ -1303,16 +1499,17 @@ static void DumpNode(CordRep* rep, bool include_data, std::ostream* os, ABSL_INTERNAL_CHECK(indents.empty(), ""); } -static std::string ReportError(CordRep* root, CordRep* node) { +static std::string ReportError(absl::Nonnull<CordRep*> root, + absl::Nonnull<CordRep*> node) { std::ostringstream buf; buf << "Error at node " << node << " in:"; DumpNode(root, true, &buf); return buf.str(); } -static bool VerifyNode(CordRep* root, CordRep* start_node, - bool /* full_validation */) { - absl::InlinedVector<CordRep*, 2> worklist; +static bool VerifyNode(absl::Nonnull<CordRep*> root, + absl::Nonnull<CordRep*> start_node) { + absl::InlinedVector<absl::Nonnull<CordRep*>, 2> worklist; worklist.push_back(start_node); do { CordRep* node = worklist.back(); diff --git a/contrib/restricted/abseil-cpp/absl/strings/cord.h b/contrib/restricted/abseil-cpp/absl/strings/cord.h index 457ccf0609..b3e556b6b4 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/cord.h +++ b/contrib/restricted/abseil-cpp/absl/strings/cord.h @@ -74,6 +74,7 @@ #include "absl/base/internal/endian.h" #include "absl/base/internal/per_thread_tls.h" #include "absl/base/macros.h" +#include "absl/base/nullability.h" #include "absl/base/port.h" #include "absl/container/inlined_vector.h" #include "absl/crc/internal/crc_cord_state.h" @@ -86,7 +87,6 @@ #include "absl/strings/internal/cord_rep_btree.h" #include "absl/strings/internal/cord_rep_btree_reader.h" #include "absl/strings/internal/cord_rep_crc.h" -#include "absl/strings/internal/cord_rep_ring.h" #include "absl/strings/internal/cordz_functions.h" #include "absl/strings/internal/cordz_info.h" #include "absl/strings/internal/cordz_statistics.h" @@ -103,7 +103,7 @@ class Cord; class CordTestPeer; template <typename Releaser> Cord MakeCordFromExternal(absl::string_view, Releaser&&); -void CopyCordToString(const Cord& src, std::string* dst); +void CopyCordToString(const Cord& src, absl::Nonnull<std::string*> dst); // Cord memory accounting modes enum class CordMemoryAccounting { @@ -120,8 +120,8 @@ enum class CordMemoryAccounting { // // For example: // absl::Cord cord; - // cord.append(some_other_cord); - // cord.append(some_other_cord); + // cord.Append(some_other_cord); + // cord.Append(some_other_cord); // // Counts `some_other_cord` twice: // cord.EstimatedMemoryUsage(kTotal); // // Counts `some_other_cord` once: @@ -362,7 +362,7 @@ class Cord { // Cord::empty() // - // Determines whether the given Cord is empty, returning `true` is so. + // Determines whether the given Cord is empty, returning `true` if so. bool empty() const; // Cord::EstimatedMemoryUsage() @@ -396,6 +396,12 @@ class Cord { bool EndsWith(absl::string_view rhs) const; bool EndsWith(const Cord& rhs) const; + // Cord::Contains() + // + // Determines whether the Cord contains the passed string data `rhs`. + bool Contains(absl::string_view rhs) const; + bool Contains(const Cord& rhs) const; + // Cord::operator std::string() // // Converts a Cord into a `std::string()`. This operator is marked explicit to @@ -411,7 +417,8 @@ class Cord { // guarantee that pointers previously returned by `dst->data()` remain valid // even if `*dst` had enough capacity to hold `src`. If `*dst` is a new // object, prefer to simply use the conversion operator to `std::string`. - friend void CopyCordToString(const Cord& src, std::string* dst); + friend void CopyCordToString(const Cord& src, + absl::Nonnull<std::string*> dst); class CharIterator; @@ -448,7 +455,7 @@ class Cord { using iterator_category = std::input_iterator_tag; using value_type = absl::string_view; using difference_type = ptrdiff_t; - using pointer = const value_type*; + using pointer = absl::Nonnull<const value_type*>; using reference = value_type; ChunkIterator() = default; @@ -468,14 +475,14 @@ class Cord { using CordRepBtree = absl::cord_internal::CordRepBtree; using CordRepBtreeReader = absl::cord_internal::CordRepBtreeReader; - // Constructs a `begin()` iterator from `tree`. `tree` must not be null. - explicit ChunkIterator(cord_internal::CordRep* tree); + // Constructs a `begin()` iterator from `tree`. + explicit ChunkIterator(absl::Nonnull<cord_internal::CordRep*> tree); // Constructs a `begin()` iterator from `cord`. - explicit ChunkIterator(const Cord* cord); + explicit ChunkIterator(absl::Nonnull<const Cord*> cord); // Initializes this instance from a tree. Invoked by constructors. - void InitTree(cord_internal::CordRep* tree); + void InitTree(absl::Nonnull<cord_internal::CordRep*> tree); // Removes `n` bytes from `current_chunk_`. Expects `n` to be smaller than // `current_chunk_.size()`. @@ -493,7 +500,7 @@ class Cord { // The current leaf, or `nullptr` if the iterator points to short data. // If the current chunk is a substring node, current_leaf_ points to the // underlying flat or external node. - absl::cord_internal::CordRep* current_leaf_ = nullptr; + absl::Nullable<absl::cord_internal::CordRep*> current_leaf_ = nullptr; // The number of bytes left in the `Cord` over which we are iterating. size_t bytes_remaining_ = 0; @@ -515,7 +522,7 @@ class Cord { // absl::string_view s) { // return std::find(c.chunk_begin(), c.chunk_end(), s); // } - ChunkIterator chunk_begin() const; + ChunkIterator chunk_begin() const ABSL_ATTRIBUTE_LIFETIME_BOUND; // Cord::chunk_end() // @@ -524,7 +531,7 @@ class Cord { // Generally, prefer using `Cord::Chunks()` within a range-based for loop for // iterating over the chunks of a Cord. This method may be useful for getting // a `ChunkIterator` where range-based for-loops may not be available. - ChunkIterator chunk_end() const; + ChunkIterator chunk_end() const ABSL_ATTRIBUTE_LIFETIME_BOUND; //---------------------------------------------------------------------------- // Cord::ChunkRange @@ -550,13 +557,13 @@ class Cord { using iterator = ChunkIterator; using const_iterator = ChunkIterator; - explicit ChunkRange(const Cord* cord) : cord_(cord) {} + explicit ChunkRange(absl::Nonnull<const Cord*> cord) : cord_(cord) {} ChunkIterator begin() const; ChunkIterator end() const; private: - const Cord* cord_; + absl::Nonnull<const Cord*> cord_; }; // Cord::Chunks() @@ -578,7 +585,7 @@ class Cord { // // The temporary Cord returned by CordFactory has been destroyed! // } // } - ChunkRange Chunks() const; + ChunkRange Chunks() const ABSL_ATTRIBUTE_LIFETIME_BOUND; //---------------------------------------------------------------------------- // Cord::CharIterator @@ -609,7 +616,7 @@ class Cord { using iterator_category = std::input_iterator_tag; using value_type = char; using difference_type = ptrdiff_t; - using pointer = const char*; + using pointer = absl::Nonnull<const char*>; using reference = const char&; CharIterator() = default; @@ -624,7 +631,8 @@ class Cord { friend Cord; private: - explicit CharIterator(const Cord* cord) : chunk_iterator_(cord) {} + explicit CharIterator(absl::Nonnull<const Cord*> cord) + : chunk_iterator_(cord) {} ChunkIterator chunk_iterator_; }; @@ -635,14 +643,14 @@ class Cord { // advanced as a separate `Cord`. `n_bytes` must be less than or equal to the // number of bytes within the Cord; otherwise, behavior is undefined. It is // valid to pass `char_end()` and `0`. - static Cord AdvanceAndRead(CharIterator* it, size_t n_bytes); + static Cord AdvanceAndRead(absl::Nonnull<CharIterator*> it, size_t n_bytes); // Cord::Advance() // // Advances the `Cord::CharIterator` by `n_bytes`. `n_bytes` must be less than // or equal to the number of bytes remaining within the Cord; otherwise, // behavior is undefined. It is valid to pass `char_end()` and `0`. - static void Advance(CharIterator* it, size_t n_bytes); + static void Advance(absl::Nonnull<CharIterator*> it, size_t n_bytes); // Cord::ChunkRemaining() // @@ -658,7 +666,7 @@ class Cord { // Generally, prefer using `Cord::Chars()` within a range-based for loop for // iterating over the chunks of a Cord. This method may be useful for getting // a `CharIterator` where range-based for-loops may not be available. - CharIterator char_begin() const; + CharIterator char_begin() const ABSL_ATTRIBUTE_LIFETIME_BOUND; // Cord::char_end() // @@ -667,7 +675,7 @@ class Cord { // Generally, prefer using `Cord::Chars()` within a range-based for loop for // iterating over the chunks of a Cord. This method may be useful for getting // a `CharIterator` where range-based for-loops are not useful. - CharIterator char_end() const; + CharIterator char_end() const ABSL_ATTRIBUTE_LIFETIME_BOUND; // Cord::CharRange // @@ -691,13 +699,13 @@ class Cord { using iterator = CharIterator; using const_iterator = CharIterator; - explicit CharRange(const Cord* cord) : cord_(cord) {} + explicit CharRange(absl::Nonnull<const Cord*> cord) : cord_(cord) {} CharIterator begin() const; CharIterator end() const; private: - const Cord* cord_; + absl::Nonnull<const Cord*> cord_; }; // Cord::Chars() @@ -719,7 +727,7 @@ class Cord { // // The temporary Cord returned by CordFactory has been destroyed! // } // } - CharRange Chars() const; + CharRange Chars() const ABSL_ATTRIBUTE_LIFETIME_BOUND; // Cord::operator[] // @@ -737,20 +745,38 @@ class Cord { // // If this cord's representation is a single flat array, returns a // string_view referencing that array. Otherwise returns nullopt. - absl::optional<absl::string_view> TryFlat() const; + absl::optional<absl::string_view> TryFlat() const + ABSL_ATTRIBUTE_LIFETIME_BOUND; // Cord::Flatten() // // Flattens the cord into a single array and returns a view of the data. // // If the cord was already flat, the contents are not modified. - absl::string_view Flatten(); + absl::string_view Flatten() ABSL_ATTRIBUTE_LIFETIME_BOUND; + + // Cord::Find() + // + // Returns an iterator to the first occurrance of the substring `needle`. + // + // If the substring `needle` does not occur, `Cord::char_end()` is returned. + CharIterator Find(absl::string_view needle) const; + CharIterator Find(const absl::Cord& needle) const; // Supports absl::Cord as a sink object for absl::Format(). - friend void AbslFormatFlush(absl::Cord* cord, absl::string_view part) { + friend void AbslFormatFlush(absl::Nonnull<absl::Cord*> cord, + absl::string_view part) { cord->Append(part); } + // Support automatic stringification with absl::StrCat and absl::StrFormat. + template <typename Sink> + friend void AbslStringify(Sink& sink, const absl::Cord& cord) { + for (absl::string_view chunk : cord.Chunks()) { + sink.Append(chunk); + } + } + // Cord::SetExpectedChecksum() // // Stores a checksum value with this non-empty cord instance, for later @@ -809,7 +835,8 @@ class Cord { friend bool operator==(const Cord& lhs, const Cord& rhs); friend bool operator==(const Cord& lhs, absl::string_view rhs); - friend const CordzInfo* GetCordzInfoForTesting(const Cord& cord); + friend absl::Nullable<const CordzInfo*> GetCordzInfoForTesting( + const Cord& cord); // Calls the provided function once for each cord chunk, in order. Unlike // Chunks(), this API will not allocate memory. @@ -836,20 +863,22 @@ class Cord { InlineRep& operator=(const InlineRep& src); InlineRep& operator=(InlineRep&& src) noexcept; - explicit constexpr InlineRep(absl::string_view sv, CordRep* rep); + explicit constexpr InlineRep(absl::string_view sv, + absl::Nullable<CordRep*> rep); - void Swap(InlineRep* rhs); - bool empty() const; + void Swap(absl::Nonnull<InlineRep*> rhs); size_t size() const; - const char* data() const; // Returns nullptr if holding pointer - void set_data(const char* data, size_t n); // Discards pointer, if any - char* set_data(size_t n); // Write data to the result + // Returns nullptr if holding pointer + absl::Nullable<const char*> data() const; + // Discards pointer, if any + void set_data(absl::Nonnull<const char*> data, size_t n); + absl::Nonnull<char*> set_data(size_t n); // Write data to the result // Returns nullptr if holding bytes - absl::cord_internal::CordRep* tree() const; - absl::cord_internal::CordRep* as_tree() const; - const char* as_chars() const; + absl::Nullable<absl::cord_internal::CordRep*> tree() const; + absl::Nonnull<absl::cord_internal::CordRep*> as_tree() const; + absl::Nonnull<const char*> as_chars() const; // Returns non-null iff was holding a pointer - absl::cord_internal::CordRep* clear(); + absl::Nullable<absl::cord_internal::CordRep*> clear(); // Converts to pointer if necessary. void reduce_size(size_t n); // REQUIRES: holding data void remove_prefix(size_t n); // REQUIRES: holding data @@ -858,46 +887,52 @@ class Cord { // Creates a CordRepFlat instance from the current inlined data with `extra' // bytes of desired additional capacity. - CordRepFlat* MakeFlatWithExtraCapacity(size_t extra); + absl::Nonnull<CordRepFlat*> MakeFlatWithExtraCapacity(size_t extra); // Sets the tree value for this instance. `rep` must not be null. // Requires the current instance to hold a tree, and a lock to be held on // any CordzInfo referenced by this instance. The latter is enforced through // the CordzUpdateScope argument. If the current instance is sampled, then // the CordzInfo instance is updated to reference the new `rep` value. - void SetTree(CordRep* rep, const CordzUpdateScope& scope); + void SetTree(absl::Nonnull<CordRep*> rep, const CordzUpdateScope& scope); // Identical to SetTree(), except that `rep` is allowed to be null, in // which case the current instance is reset to an empty value. - void SetTreeOrEmpty(CordRep* rep, const CordzUpdateScope& scope); + void SetTreeOrEmpty(absl::Nullable<CordRep*> rep, + const CordzUpdateScope& scope); // Sets the tree value for this instance, and randomly samples this cord. // This function disregards existing contents in `data_`, and should be // called when a Cord is 'promoted' from an 'uninitialized' or 'inlined' // value to a non-inlined (tree / ring) value. - void EmplaceTree(CordRep* rep, MethodIdentifier method); + void EmplaceTree(absl::Nonnull<CordRep*> rep, MethodIdentifier method); // Identical to EmplaceTree, except that it copies the parent stack from // the provided `parent` data if the parent is sampled. - void EmplaceTree(CordRep* rep, const InlineData& parent, + void EmplaceTree(absl::Nonnull<CordRep*> rep, const InlineData& parent, MethodIdentifier method); // Commits the change of a newly created, or updated `rep` root value into // this cord. `old_rep` indicates the old (inlined or tree) value of the // cord, and determines if the commit invokes SetTree() or EmplaceTree(). - void CommitTree(const CordRep* old_rep, CordRep* rep, - const CordzUpdateScope& scope, MethodIdentifier method); - - void AppendTreeToInlined(CordRep* tree, MethodIdentifier method); - void AppendTreeToTree(CordRep* tree, MethodIdentifier method); - void AppendTree(CordRep* tree, MethodIdentifier method); - void PrependTreeToInlined(CordRep* tree, MethodIdentifier method); - void PrependTreeToTree(CordRep* tree, MethodIdentifier method); - void PrependTree(CordRep* tree, MethodIdentifier method); + void CommitTree(absl::Nullable<const CordRep*> old_rep, + absl::Nonnull<CordRep*> rep, const CordzUpdateScope& scope, + MethodIdentifier method); + + void AppendTreeToInlined(absl::Nonnull<CordRep*> tree, + MethodIdentifier method); + void AppendTreeToTree(absl::Nonnull<CordRep*> tree, + MethodIdentifier method); + void AppendTree(absl::Nonnull<CordRep*> tree, MethodIdentifier method); + void PrependTreeToInlined(absl::Nonnull<CordRep*> tree, + MethodIdentifier method); + void PrependTreeToTree(absl::Nonnull<CordRep*> tree, + MethodIdentifier method); + void PrependTree(absl::Nonnull<CordRep*> tree, MethodIdentifier method); bool IsSame(const InlineRep& other) const { return data_ == other.data_; } - void CopyTo(std::string* dst) const { + void CopyTo(absl::Nonnull<std::string*> dst) const { // memcpy is much faster when operating on a known size. On most supported // platforms, the small string optimization is large enough that resizing // to 15 bytes does not cause a memory allocation. @@ -909,7 +944,7 @@ class Cord { } // Copies the inline contents into `dst`. Assumes the cord is not empty. - void CopyToArray(char* dst) const; + void CopyToArray(absl::Nonnull<char*> dst) const; bool is_tree() const { return data_.is_tree(); } @@ -922,12 +957,12 @@ class Cord { } // Returns the profiled CordzInfo, or nullptr if not sampled. - absl::cord_internal::CordzInfo* cordz_info() const { + absl::Nullable<absl::cord_internal::CordzInfo*> cordz_info() const { return data_.cordz_info(); } - // Sets the profiled CordzInfo. `cordz_info` must not be null. - void set_cordz_info(cord_internal::CordzInfo* cordz_info) { + // Sets the profiled CordzInfo. + void set_cordz_info(absl::Nonnull<cord_internal::CordzInfo*> cordz_info) { assert(cordz_info != nullptr); data_.set_cordz_info(cordz_info); } @@ -959,19 +994,19 @@ class Cord { InlineRep contents_; // Helper for GetFlat() and TryFlat(). - static bool GetFlatAux(absl::cord_internal::CordRep* rep, - absl::string_view* fragment); + static bool GetFlatAux(absl::Nonnull<absl::cord_internal::CordRep*> rep, + absl::Nonnull<absl::string_view*> fragment); // Helper for ForEachChunk(). static void ForEachChunkAux( - absl::cord_internal::CordRep* rep, + absl::Nonnull<absl::cord_internal::CordRep*> rep, absl::FunctionRef<void(absl::string_view)> callback); // The destructor for non-empty Cords. void DestroyCordSlow(); // Out-of-line implementation of slower parts of logic. - void CopyToArraySlowPath(char* dst) const; + void CopyToArraySlowPath(absl::Nonnull<char*> dst) const; int CompareSlowPath(absl::string_view rhs, size_t compared_size, size_t size_to_compare) const; int CompareSlowPath(const Cord& rhs, size_t compared_size, @@ -988,8 +1023,8 @@ class Cord { // Returns a new reference to contents_.tree(), or steals an existing // reference if called on an rvalue. - absl::cord_internal::CordRep* TakeRep() const&; - absl::cord_internal::CordRep* TakeRep() &&; + absl::Nonnull<absl::cord_internal::CordRep*> TakeRep() const&; + absl::Nonnull<absl::cord_internal::CordRep*> TakeRep() &&; // Helper for Append(). template <typename C> @@ -1026,7 +1061,10 @@ class Cord { friend class CrcCord; void SetCrcCordState(crc_internal::CrcCordState state); - const crc_internal::CrcCordState* MaybeGetCrcCordState() const; + absl::Nullable<const crc_internal::CrcCordState*> MaybeGetCrcCordState() + const; + + CharIterator FindImpl(CharIterator it, absl::string_view needle) const; }; ABSL_NAMESPACE_END @@ -1045,13 +1083,15 @@ namespace cord_internal { // Does non-template-specific `CordRepExternal` initialization. // Requires `data` to be non-empty. -void InitializeCordRepExternal(absl::string_view data, CordRepExternal* rep); +void InitializeCordRepExternal(absl::string_view data, + absl::Nonnull<CordRepExternal*> rep); // Creates a new `CordRep` that owns `data` and `releaser` and returns a pointer // to it. Requires `data` to be non-empty. template <typename Releaser> // NOLINTNEXTLINE - suppress clang-tidy raw pointer return. -CordRep* NewExternalRep(absl::string_view data, Releaser&& releaser) { +absl::Nonnull<CordRep*> NewExternalRep(absl::string_view data, + Releaser&& releaser) { assert(!data.empty()); using ReleaserType = absl::decay_t<Releaser>; CordRepExternal* rep = new CordRepExternalImpl<ReleaserType>( @@ -1063,7 +1103,7 @@ CordRep* NewExternalRep(absl::string_view data, Releaser&& releaser) { // Overload for function reference types that dispatches using a function // pointer because there are no `alignof()` or `sizeof()` a function reference. // NOLINTNEXTLINE - suppress clang-tidy raw pointer return. -inline CordRep* NewExternalRep(absl::string_view data, +inline absl::Nonnull<CordRep*> NewExternalRep(absl::string_view data, void (&releaser)(absl::string_view)) { return NewExternalRep(data, &releaser); } @@ -1086,7 +1126,8 @@ Cord MakeCordFromExternal(absl::string_view data, Releaser&& releaser) { return cord; } -constexpr Cord::InlineRep::InlineRep(absl::string_view sv, CordRep* rep) +constexpr Cord::InlineRep::InlineRep(absl::string_view sv, + absl::Nullable<CordRep*> rep) : data_(sv, rep) {} inline Cord::InlineRep::InlineRep(const Cord::InlineRep& src) @@ -1125,28 +1166,30 @@ inline Cord::InlineRep& Cord::InlineRep::operator=( return *this; } -inline void Cord::InlineRep::Swap(Cord::InlineRep* rhs) { +inline void Cord::InlineRep::Swap(absl::Nonnull<Cord::InlineRep*> rhs) { if (rhs == this) { return; } std::swap(data_, rhs->data_); } -inline const char* Cord::InlineRep::data() const { +inline absl::Nullable<const char*> Cord::InlineRep::data() const { return is_tree() ? nullptr : data_.as_chars(); } -inline const char* Cord::InlineRep::as_chars() const { +inline absl::Nonnull<const char*> Cord::InlineRep::as_chars() const { assert(!data_.is_tree()); return data_.as_chars(); } -inline absl::cord_internal::CordRep* Cord::InlineRep::as_tree() const { +inline absl::Nonnull<absl::cord_internal::CordRep*> Cord::InlineRep::as_tree() + const { assert(data_.is_tree()); return data_.as_tree(); } -inline absl::cord_internal::CordRep* Cord::InlineRep::tree() const { +inline absl::Nullable<absl::cord_internal::CordRep*> Cord::InlineRep::tree() + const { if (is_tree()) { return as_tree(); } else { @@ -1154,14 +1197,12 @@ inline absl::cord_internal::CordRep* Cord::InlineRep::tree() const { } } -inline bool Cord::InlineRep::empty() const { return data_.is_empty(); } - inline size_t Cord::InlineRep::size() const { return is_tree() ? as_tree()->length : inline_size(); } -inline cord_internal::CordRepFlat* Cord::InlineRep::MakeFlatWithExtraCapacity( - size_t extra) { +inline absl::Nonnull<cord_internal::CordRepFlat*> +Cord::InlineRep::MakeFlatWithExtraCapacity(size_t extra) { static_assert(cord_internal::kMinFlatLength >= sizeof(data_), ""); size_t len = data_.inline_size(); auto* result = CordRepFlat::New(len + extra); @@ -1170,20 +1211,21 @@ inline cord_internal::CordRepFlat* Cord::InlineRep::MakeFlatWithExtraCapacity( return result; } -inline void Cord::InlineRep::EmplaceTree(CordRep* rep, +inline void Cord::InlineRep::EmplaceTree(absl::Nonnull<CordRep*> rep, MethodIdentifier method) { assert(rep); data_.make_tree(rep); CordzInfo::MaybeTrackCord(data_, method); } -inline void Cord::InlineRep::EmplaceTree(CordRep* rep, const InlineData& parent, +inline void Cord::InlineRep::EmplaceTree(absl::Nonnull<CordRep*> rep, + const InlineData& parent, MethodIdentifier method) { data_.make_tree(rep); CordzInfo::MaybeTrackCord(data_, parent, method); } -inline void Cord::InlineRep::SetTree(CordRep* rep, +inline void Cord::InlineRep::SetTree(absl::Nonnull<CordRep*> rep, const CordzUpdateScope& scope) { assert(rep); assert(data_.is_tree()); @@ -1191,7 +1233,7 @@ inline void Cord::InlineRep::SetTree(CordRep* rep, scope.SetCordRep(rep); } -inline void Cord::InlineRep::SetTreeOrEmpty(CordRep* rep, +inline void Cord::InlineRep::SetTreeOrEmpty(absl::Nullable<CordRep*> rep, const CordzUpdateScope& scope) { assert(data_.is_tree()); if (rep) { @@ -1202,7 +1244,8 @@ inline void Cord::InlineRep::SetTreeOrEmpty(CordRep* rep, scope.SetCordRep(rep); } -inline void Cord::InlineRep::CommitTree(const CordRep* old_rep, CordRep* rep, +inline void Cord::InlineRep::CommitTree(absl::Nullable<const CordRep*> old_rep, + absl::Nonnull<CordRep*> rep, const CordzUpdateScope& scope, MethodIdentifier method) { if (old_rep) { @@ -1212,7 +1255,7 @@ inline void Cord::InlineRep::CommitTree(const CordRep* old_rep, CordRep* rep, } } -inline absl::cord_internal::CordRep* Cord::InlineRep::clear() { +inline absl::Nullable<absl::cord_internal::CordRep*> Cord::InlineRep::clear() { if (is_tree()) { CordzInfo::MaybeUntrackCord(cordz_info()); } @@ -1221,7 +1264,7 @@ inline absl::cord_internal::CordRep* Cord::InlineRep::clear() { return result; } -inline void Cord::InlineRep::CopyToArray(char* dst) const { +inline void Cord::InlineRep::CopyToArray(absl::Nonnull<char*> dst) const { assert(!is_tree()); size_t n = inline_size(); assert(n != 0); @@ -1402,7 +1445,8 @@ inline bool Cord::StartsWith(absl::string_view rhs) const { return EqualsImpl(rhs, rhs_size); } -inline void Cord::ChunkIterator::InitTree(cord_internal::CordRep* tree) { +inline void Cord::ChunkIterator::InitTree( + absl::Nonnull<cord_internal::CordRep*> tree) { tree = cord_internal::SkipCrcNode(tree); if (tree->tag == cord_internal::BTREE) { current_chunk_ = btree_reader_.Init(tree->btree()); @@ -1412,12 +1456,13 @@ inline void Cord::ChunkIterator::InitTree(cord_internal::CordRep* tree) { } } -inline Cord::ChunkIterator::ChunkIterator(cord_internal::CordRep* tree) { +inline Cord::ChunkIterator::ChunkIterator( + absl::Nonnull<cord_internal::CordRep*> tree) { bytes_remaining_ = tree->length; InitTree(tree); } -inline Cord::ChunkIterator::ChunkIterator(const Cord* cord) { +inline Cord::ChunkIterator::ChunkIterator(absl::Nonnull<const Cord*> cord) { if (CordRep* tree = cord->contents_.tree()) { bytes_remaining_ = tree->length; if (ABSL_PREDICT_TRUE(bytes_remaining_ != 0)) { @@ -1557,12 +1602,13 @@ inline Cord::CharIterator::pointer Cord::CharIterator::operator->() const { return chunk_iterator_->data(); } -inline Cord Cord::AdvanceAndRead(CharIterator* it, size_t n_bytes) { +inline Cord Cord::AdvanceAndRead(absl::Nonnull<CharIterator*> it, + size_t n_bytes) { assert(it != nullptr); return it->chunk_iterator_.AdvanceAndReadBytes(n_bytes); } -inline void Cord::Advance(CharIterator* it, size_t n_bytes) { +inline void Cord::Advance(absl::Nonnull<CharIterator*> it, size_t n_bytes) { assert(it != nullptr); it->chunk_iterator_.AdvanceBytes(n_bytes); } @@ -1618,7 +1664,7 @@ inline bool operator>=(const Cord& x, const Cord& y) { // Nonmember Cord-to-absl::string_view relational operators. // // Due to implicit conversions, these also enable comparisons of Cord with -// with std::string, ::string, and const char*. +// std::string and const char*. inline bool operator==(const Cord& lhs, absl::string_view rhs) { size_t lhs_size = lhs.size(); size_t rhs_size = rhs.size(); diff --git a/contrib/restricted/abseil-cpp/absl/strings/cord_analysis.cc b/contrib/restricted/abseil-cpp/absl/strings/cord_analysis.cc index e859b0db06..19b0fa4436 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/cord_analysis.cc +++ b/contrib/restricted/abseil-cpp/absl/strings/cord_analysis.cc @@ -14,23 +14,17 @@ #include "absl/strings/cord_analysis.h" +#include <cassert> #include <cstddef> #include <cstdint> #include <unordered_set> -#include "absl/base/attributes.h" #include "absl/base/config.h" -#include "absl/container/inlined_vector.h" +#include "absl/base/nullability.h" #include "absl/strings/internal/cord_data_edge.h" #include "absl/strings/internal/cord_internal.h" #include "absl/strings/internal/cord_rep_btree.h" #include "absl/strings/internal/cord_rep_crc.h" -#include "absl/strings/internal/cord_rep_flat.h" -#include "absl/strings/internal/cord_rep_ring.h" -// -#include "absl/base/macros.h" -#include "absl/base/port.h" -#include "absl/functional/function_ref.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -45,13 +39,15 @@ enum class Mode { kFairShare, kTotal, kTotalMorePrecise }; template <Mode mode> struct CordRepRef { // Instantiates a CordRepRef instance. - explicit CordRepRef(const CordRep* r) : rep(r) {} + explicit CordRepRef(absl::Nonnull<const CordRep*> r) : rep(r) {} // Creates a child reference holding the provided child. // Overloaded to add cumulative reference count for kFairShare. - CordRepRef Child(const CordRep* child) const { return CordRepRef(child); } + CordRepRef Child(absl::Nonnull<const CordRep*> child) const { + return CordRepRef(child); + } - const CordRep* rep; + absl::Nonnull<const CordRep*> rep; }; // RawUsage holds the computed total number of bytes. @@ -70,11 +66,10 @@ template <> struct RawUsage<Mode::kTotalMorePrecise> { size_t total = 0; // TODO(b/289250880): Replace this with a flat_hash_set. - std::unordered_set<const CordRep*> counted; + std::unordered_set<absl::Nonnull<const CordRep*>> counted; void Add(size_t size, CordRepRef<Mode::kTotalMorePrecise> repref) { - if (counted.find(repref.rep) == counted.end()) { - counted.insert(repref.rep); + if (counted.insert(repref.rep).second) { total += size; } } @@ -95,15 +90,15 @@ double MaybeDiv(double d, refcount_t refcount) { template <> struct CordRepRef<Mode::kFairShare> { // Creates a CordRepRef with the provided rep and top (parent) fraction. - explicit CordRepRef(const CordRep* r, double frac = 1.0) + explicit CordRepRef(absl::Nonnull<const CordRep*> r, double frac = 1.0) : rep(r), fraction(MaybeDiv(frac, r->refcount.Get())) {} // Returns a CordRepRef with a fraction of `this->fraction / child.refcount` - CordRepRef Child(const CordRep* child) const { + CordRepRef Child(absl::Nonnull<const CordRep*> child) const { return CordRepRef(child, fraction); } - const CordRep* rep; + absl::Nonnull<const CordRep*> rep; double fraction; }; @@ -138,16 +133,6 @@ void AnalyzeDataEdge(CordRepRef<mode> rep, RawUsage<mode>& raw_usage) { raw_usage.Add(size, rep); } -// Computes the memory size of the provided Ring tree. -template <Mode mode> -void AnalyzeRing(CordRepRef<mode> rep, RawUsage<mode>& raw_usage) { - const CordRepRing* ring = rep.rep->ring(); - raw_usage.Add(CordRepRing::AllocSize(ring->capacity()), rep); - ring->ForEach([&](CordRepRing::index_type pos) { - AnalyzeDataEdge(rep.Child(ring->entry_child(pos)), raw_usage); - }); -} - // Computes the memory size of the provided Btree tree. template <Mode mode> void AnalyzeBtree(CordRepRef<mode> rep, RawUsage<mode>& raw_usage) { @@ -165,7 +150,7 @@ void AnalyzeBtree(CordRepRef<mode> rep, RawUsage<mode>& raw_usage) { } template <Mode mode> -size_t GetEstimatedUsage(const CordRep* rep) { +size_t GetEstimatedUsage(absl::Nonnull<const CordRep*> rep) { // Zero initialized memory usage totals. RawUsage<mode> raw_usage; @@ -175,6 +160,9 @@ size_t GetEstimatedUsage(const CordRep* rep) { // Consume the top level CRC node if present. if (repref.rep->tag == CRC) { raw_usage.Add(sizeof(CordRepCrc), repref); + if (repref.rep->crc()->child == nullptr) { + return static_cast<size_t>(raw_usage.total); + } repref = repref.Child(repref.rep->crc()->child); } @@ -182,8 +170,6 @@ size_t GetEstimatedUsage(const CordRep* rep) { AnalyzeDataEdge(repref, raw_usage); } else if (repref.rep->tag == BTREE) { AnalyzeBtree(repref, raw_usage); - } else if (repref.rep->tag == RING) { - AnalyzeRing(repref, raw_usage); } else { assert(false); } @@ -193,15 +179,15 @@ size_t GetEstimatedUsage(const CordRep* rep) { } // namespace -size_t GetEstimatedMemoryUsage(const CordRep* rep) { +size_t GetEstimatedMemoryUsage(absl::Nonnull<const CordRep*> rep) { return GetEstimatedUsage<Mode::kTotal>(rep); } -size_t GetEstimatedFairShareMemoryUsage(const CordRep* rep) { +size_t GetEstimatedFairShareMemoryUsage(absl::Nonnull<const CordRep*> rep) { return GetEstimatedUsage<Mode::kFairShare>(rep); } -size_t GetMorePreciseMemoryUsage(const CordRep* rep) { +size_t GetMorePreciseMemoryUsage(absl::Nonnull<const CordRep*> rep) { return GetEstimatedUsage<Mode::kTotalMorePrecise>(rep); } diff --git a/contrib/restricted/abseil-cpp/absl/strings/cord_analysis.h b/contrib/restricted/abseil-cpp/absl/strings/cord_analysis.h index 9b9527a57d..f8ce348984 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/cord_analysis.h +++ b/contrib/restricted/abseil-cpp/absl/strings/cord_analysis.h @@ -19,6 +19,7 @@ #include <cstdint> #include "absl/base/config.h" +#include "absl/base/nullability.h" #include "absl/strings/internal/cord_internal.h" namespace absl { @@ -28,7 +29,7 @@ namespace cord_internal { // Returns the *approximate* number of bytes held in full or in part by this // Cord (which may not remain the same between invocations). Cords that share // memory could each be "charged" independently for the same shared memory. -size_t GetEstimatedMemoryUsage(const CordRep* rep); +size_t GetEstimatedMemoryUsage(absl::Nonnull<const CordRep*> rep); // Returns the *approximate* number of bytes held in full or in part by this // Cord for the distinct memory held by this cord. This is similar to @@ -46,13 +47,13 @@ size_t GetEstimatedMemoryUsage(const CordRep* rep); // // This is more expensive than `GetEstimatedMemoryUsage()` as it requires // deduplicating all memory references. -size_t GetMorePreciseMemoryUsage(const CordRep* rep); +size_t GetMorePreciseMemoryUsage(absl::Nonnull<const CordRep*> rep); // Returns the *approximate* number of bytes held in full or in part by this // CordRep weighted by the sharing ratio of that data. For example, if some data // edge is shared by 4 different Cords, then each cord is attribute 1/4th of // the total memory usage as a 'fair share' of the total memory usage. -size_t GetEstimatedFairShareMemoryUsage(const CordRep* rep); +size_t GetEstimatedFairShareMemoryUsage(absl::Nonnull<const CordRep*> rep); } // namespace cord_internal ABSL_NAMESPACE_END diff --git a/contrib/restricted/abseil-cpp/absl/strings/escaping.cc b/contrib/restricted/abseil-cpp/absl/strings/escaping.cc index 2827fbaa37..1c0eac42d7 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/escaping.cc +++ b/contrib/restricted/abseil-cpp/absl/strings/escaping.cc @@ -16,21 +16,22 @@ #include <algorithm> #include <cassert> +#include <cstddef> #include <cstdint> #include <cstring> -#include <iterator> #include <limits> #include <string> -#include "absl/base/internal/endian.h" +#include "absl/base/config.h" #include "absl/base/internal/raw_logging.h" #include "absl/base/internal/unaligned_access.h" -#include "absl/strings/internal/char_map.h" +#include "absl/strings/ascii.h" +#include "absl/strings/charset.h" #include "absl/strings/internal/escaping.h" #include "absl/strings/internal/resize_uninitialized.h" #include "absl/strings/internal/utf8.h" +#include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" -#include "absl/strings/str_join.h" #include "absl/strings/string_view.h" namespace absl { diff --git a/contrib/restricted/abseil-cpp/absl/strings/internal/has_absl_stringify.h b/contrib/restricted/abseil-cpp/absl/strings/has_absl_stringify.h index 55a0850829..274a7865d1 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/internal/has_absl_stringify.h +++ b/contrib/restricted/abseil-cpp/absl/strings/has_absl_stringify.h @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef ABSL_STRINGS_INTERNAL_HAS_ABSL_STRINGIFY_H_ -#define ABSL_STRINGS_INTERNAL_HAS_ABSL_STRINGIFY_H_ -#include <string> +#ifndef ABSL_STRINGS_HAS_ABSL_STRINGIFY_H_ +#define ABSL_STRINGS_HAS_ABSL_STRINGIFY_H_ + #include <type_traits> #include <utility> @@ -38,6 +38,16 @@ class UnimplementedSink { friend void AbslFormatFlush(UnimplementedSink* sink, absl::string_view v); }; +} // namespace strings_internal + +// `HasAbslStringify<T>` detects if type `T` supports the `AbslStringify()` +// customization point (see +// https://abseil.io/docs/cpp/guides/format#abslstringify for the +// documentation). +// +// Note that there are types that can be `StrCat`-ed that do not use the +// `AbslStringify` customization point (for example, `int`). + template <typename T, typename = void> struct HasAbslStringify : std::false_type {}; @@ -47,9 +57,7 @@ struct HasAbslStringify< std::declval<strings_internal::UnimplementedSink&>(), std::declval<const T&>()))>::value>> : std::true_type {}; -} // namespace strings_internal - ABSL_NAMESPACE_END } // namespace absl -#endif // ABSL_STRINGS_INTERNAL_HAS_ABSL_STRINGIFY_H_ +#endif // ABSL_STRINGS_HAS_ABSL_STRINGIFY_H_ diff --git a/contrib/restricted/abseil-cpp/absl/strings/has_ostream_operator.h b/contrib/restricted/abseil-cpp/absl/strings/has_ostream_operator.h new file mode 100644 index 0000000000..156ffc743d --- /dev/null +++ b/contrib/restricted/abseil-cpp/absl/strings/has_ostream_operator.h @@ -0,0 +1,42 @@ +// Copyright 2023 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_HAS_OSTREAM_OPERATOR_H_ +#define ABSL_STRINGS_HAS_OSTREAM_OPERATOR_H_ + +#include <ostream> +#include <type_traits> +#include <utility> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// Detects if type `T` supports streaming to `std::ostream`s with `operator<<`. + +template <typename T, typename = void> +struct HasOstreamOperator : std::false_type {}; + +template <typename T> +struct HasOstreamOperator< + T, std::enable_if_t<std::is_same< + std::ostream&, decltype(std::declval<std::ostream&>() + << std::declval<const T&>())>::value>> + : std::true_type {}; + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_HAS_OSTREAM_OPERATOR_H_ diff --git a/contrib/restricted/abseil-cpp/absl/strings/internal/char_map.h b/contrib/restricted/abseil-cpp/absl/strings/internal/char_map.h deleted file mode 100644 index 70a9034336..0000000000 --- a/contrib/restricted/abseil-cpp/absl/strings/internal/char_map.h +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2017 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Character Map Class -// -// A fast, bit-vector map for 8-bit unsigned characters. -// This class is useful for non-character purposes as well. - -#ifndef ABSL_STRINGS_INTERNAL_CHAR_MAP_H_ -#define ABSL_STRINGS_INTERNAL_CHAR_MAP_H_ - -#include <cstddef> -#include <cstdint> -#include <cstring> - -#include "absl/base/macros.h" -#include "absl/base/port.h" - -namespace absl { -ABSL_NAMESPACE_BEGIN -namespace strings_internal { - -class Charmap { - public: - constexpr Charmap() : m_() {} - - // Initializes with a given char*. Note that NUL is not treated as - // a terminator, but rather a char to be flicked. - Charmap(const char* str, int len) : m_() { - while (len--) SetChar(*str++); - } - - // Initializes with a given char*. NUL is treated as a terminator - // and will not be in the charmap. - explicit Charmap(const char* str) : m_() { - while (*str) SetChar(*str++); - } - - constexpr bool contains(unsigned char c) const { - return (m_[c / 64] >> (c % 64)) & 0x1; - } - - // Returns true if and only if a character exists in both maps. - bool IntersectsWith(const Charmap& c) const { - for (size_t i = 0; i < ABSL_ARRAYSIZE(m_); ++i) { - if ((m_[i] & c.m_[i]) != 0) return true; - } - return false; - } - - bool IsZero() const { - for (uint64_t c : m_) { - if (c != 0) return false; - } - return true; - } - - // Containing only a single specified char. - static constexpr Charmap Char(char x) { - return Charmap(CharMaskForWord(x, 0), CharMaskForWord(x, 1), - CharMaskForWord(x, 2), CharMaskForWord(x, 3)); - } - - // Containing all the chars in the C-string 's'. - static constexpr Charmap FromString(const char* s) { - Charmap ret; - while (*s) ret = ret | Char(*s++); - return ret; - } - - // Containing all the chars in the closed interval [lo,hi]. - static constexpr Charmap Range(char lo, char hi) { - return Charmap(RangeForWord(lo, hi, 0), RangeForWord(lo, hi, 1), - RangeForWord(lo, hi, 2), RangeForWord(lo, hi, 3)); - } - - friend constexpr Charmap operator&(const Charmap& a, const Charmap& b) { - return Charmap(a.m_[0] & b.m_[0], a.m_[1] & b.m_[1], a.m_[2] & b.m_[2], - a.m_[3] & b.m_[3]); - } - - friend constexpr Charmap operator|(const Charmap& a, const Charmap& b) { - return Charmap(a.m_[0] | b.m_[0], a.m_[1] | b.m_[1], a.m_[2] | b.m_[2], - a.m_[3] | b.m_[3]); - } - - friend constexpr Charmap operator~(const Charmap& a) { - return Charmap(~a.m_[0], ~a.m_[1], ~a.m_[2], ~a.m_[3]); - } - - private: - constexpr Charmap(uint64_t b0, uint64_t b1, uint64_t b2, uint64_t b3) - : m_{b0, b1, b2, b3} {} - - static constexpr uint64_t RangeForWord(char lo, char hi, uint64_t word) { - return OpenRangeFromZeroForWord(static_cast<unsigned char>(hi) + 1, word) & - ~OpenRangeFromZeroForWord(static_cast<unsigned char>(lo), word); - } - - // All the chars in the specified word of the range [0, upper). - static constexpr uint64_t OpenRangeFromZeroForWord(uint64_t upper, - uint64_t word) { - return (upper <= 64 * word) - ? 0 - : (upper >= 64 * (word + 1)) - ? ~static_cast<uint64_t>(0) - : (~static_cast<uint64_t>(0) >> (64 - upper % 64)); - } - - static constexpr uint64_t CharMaskForWord(char x, uint64_t word) { - const auto unsigned_x = static_cast<unsigned char>(x); - return (unsigned_x / 64 == word) - ? (static_cast<uint64_t>(1) << (unsigned_x % 64)) - : 0; - } - - void SetChar(char c) { - const auto unsigned_c = static_cast<unsigned char>(c); - m_[unsigned_c / 64] |= static_cast<uint64_t>(1) << (unsigned_c % 64); - } - - uint64_t m_[4]; -}; - -// Mirror the char-classifying predicates in <cctype> -constexpr Charmap UpperCharmap() { return Charmap::Range('A', 'Z'); } -constexpr Charmap LowerCharmap() { return Charmap::Range('a', 'z'); } -constexpr Charmap DigitCharmap() { return Charmap::Range('0', '9'); } -constexpr Charmap AlphaCharmap() { return LowerCharmap() | UpperCharmap(); } -constexpr Charmap AlnumCharmap() { return DigitCharmap() | AlphaCharmap(); } -constexpr Charmap XDigitCharmap() { - return DigitCharmap() | Charmap::Range('A', 'F') | Charmap::Range('a', 'f'); -} -constexpr Charmap PrintCharmap() { return Charmap::Range(0x20, 0x7e); } -constexpr Charmap SpaceCharmap() { return Charmap::FromString("\t\n\v\f\r "); } -constexpr Charmap CntrlCharmap() { - return Charmap::Range(0, 0x7f) & ~PrintCharmap(); -} -constexpr Charmap BlankCharmap() { return Charmap::FromString("\t "); } -constexpr Charmap GraphCharmap() { return PrintCharmap() & ~SpaceCharmap(); } -constexpr Charmap PunctCharmap() { return GraphCharmap() & ~AlnumCharmap(); } - -} // namespace strings_internal -ABSL_NAMESPACE_END -} // namespace absl - -#endif // ABSL_STRINGS_INTERNAL_CHAR_MAP_H_ diff --git a/contrib/restricted/abseil-cpp/absl/strings/internal/cord_internal.cc b/contrib/restricted/abseil-cpp/absl/strings/internal/cord_internal.cc index b7874385de..57d9d385b3 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/internal/cord_internal.cc +++ b/contrib/restricted/abseil-cpp/absl/strings/internal/cord_internal.cc @@ -22,15 +22,12 @@ #include "absl/strings/internal/cord_rep_btree.h" #include "absl/strings/internal/cord_rep_crc.h" #include "absl/strings/internal/cord_rep_flat.h" -#include "absl/strings/internal/cord_rep_ring.h" #include "absl/strings/str_cat.h" namespace absl { ABSL_NAMESPACE_BEGIN namespace cord_internal { -ABSL_CONST_INIT std::atomic<bool> cord_ring_buffer_enabled( - kCordEnableRingBufferDefault); ABSL_CONST_INIT std::atomic<bool> shallow_subcords_enabled( kCordShallowSubcordsDefault); @@ -47,9 +44,6 @@ void CordRep::Destroy(CordRep* rep) { if (rep->tag == BTREE) { CordRepBtree::Destroy(rep->btree()); return; - } else if (rep->tag == RING) { - CordRepRing::Destroy(rep->ring()); - return; } else if (rep->tag == EXTERNAL) { CordRepExternal::Delete(rep); return; diff --git a/contrib/restricted/abseil-cpp/absl/strings/internal/cord_internal.h b/contrib/restricted/abseil-cpp/absl/strings/internal/cord_internal.h index 20dd008cfa..8744540e2a 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/internal/cord_internal.h +++ b/contrib/restricted/abseil-cpp/absl/strings/internal/cord_internal.h @@ -55,24 +55,15 @@ struct CordRepExternal; struct CordRepFlat; struct CordRepSubstring; struct CordRepCrc; -class CordRepRing; class CordRepBtree; class CordzInfo; // Default feature enable states for cord ring buffers -enum CordFeatureDefaults { - kCordEnableRingBufferDefault = false, - kCordShallowSubcordsDefault = false -}; +enum CordFeatureDefaults { kCordShallowSubcordsDefault = false }; -extern std::atomic<bool> cord_ring_buffer_enabled; extern std::atomic<bool> shallow_subcords_enabled; -inline void enable_cord_ring_buffer(bool enable) { - cord_ring_buffer_enabled.store(enable, std::memory_order_relaxed); -} - inline void enable_shallow_subcords(bool enable) { shallow_subcords_enabled.store(enable, std::memory_order_relaxed); } @@ -110,8 +101,16 @@ inline void SmallMemmove(char* dst, const char* src, size_t n) { if (nullify_tail) { memset(dst + 7, 0, 8); } + // GCC 12 has a false-positive -Wstringop-overflow warning here. +#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(12, 0) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wstringop-overflow" +#endif memcpy(dst, &buf1, 8); memcpy(dst + n - 8, &buf2, 8); +#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(12, 0) +#pragma GCC diagnostic pop +#endif } else if (n >= 4) { uint32_t buf1; uint32_t buf2; @@ -158,18 +157,18 @@ class RefcountAndFlags { // false. Always returns false when the immortal bit is set. inline bool Decrement() { int32_t refcount = count_.load(std::memory_order_acquire); - assert((refcount & kRefcountMask) > 0 || refcount & kImmortalFlag); + assert(refcount > 0 || refcount & kImmortalFlag); return refcount != kRefIncrement && - (count_.fetch_sub(kRefIncrement, std::memory_order_acq_rel) & - kHighRefcountMask) != 0; + count_.fetch_sub(kRefIncrement, std::memory_order_acq_rel) != + kRefIncrement; } // Same as Decrement but expect that refcount is greater than 1. inline bool DecrementExpectHighRefcount() { int32_t refcount = count_.fetch_sub(kRefIncrement, std::memory_order_acq_rel); - assert((refcount & kRefcountMask) > 0 || refcount & kImmortalFlag); - return (refcount & kHighRefcountMask) != 0; + assert(refcount > 0 || refcount & kImmortalFlag); + return refcount != kRefIncrement; } // Returns the current reference count using acquire semantics. @@ -185,10 +184,9 @@ class RefcountAndFlags { // This call performs the test for a reference count of one, and // performs the memory barrier needed for the owning thread // to act on the object, knowing that it has exclusive access to the - // object. Always returns false when the immortal bit is set. + // object. Always returns false when the immortal bit is set. inline bool IsOne() { - return (count_.load(std::memory_order_acquire) & kRefcountMask) == - kRefIncrement; + return count_.load(std::memory_order_acquire) == kRefIncrement; } bool IsImmortal() const { @@ -196,32 +194,15 @@ class RefcountAndFlags { } private: - // We reserve the bottom bits for flags. + // We reserve the bottom bit for flag. // kImmortalBit indicates that this entity should never be collected; it is // used for the StringConstant constructor to avoid collecting immutable // constant cords. - // kReservedFlag is reserved for future use. enum Flags { - kNumFlags = 2, + kNumFlags = 1, kImmortalFlag = 0x1, - kReservedFlag = 0x2, kRefIncrement = (1 << kNumFlags), - - // Bitmask to use when checking refcount by equality. This masks out - // all flags except kImmortalFlag, which is part of the refcount for - // purposes of equality. (A refcount of 0 or 1 does not count as 0 or 1 - // if the immortal bit is set.) - kRefcountMask = ~kReservedFlag, - - // Bitmask to use when checking if refcount is equal to 1 and not - // immortal when decrementing the refcount. This masks out kRefIncrement and - // all flags except kImmortalFlag. If the masked RefcountAndFlags is 0, we - // assume the refcount is equal to 1, since we know it's not immortal and - // not greater than 1. If the masked RefcountAndFlags is not 0, we can - // assume the refcount is not equal to 1 since either a higher bit in the - // refcount is set, or kImmortal is set. - kHighRefcountMask = kRefcountMask & ~kRefIncrement, }; std::atomic<int32_t> count_; @@ -233,7 +214,7 @@ enum CordRepKind { SUBSTRING = 1, CRC = 2, BTREE = 3, - RING = 4, + UNUSED_4 = 4, EXTERNAL = 5, // We have different tags for different sized flat arrays, @@ -252,12 +233,8 @@ enum CordRepKind { // There are various locations where we want to check if some rep is a 'plain' // data edge, i.e. an external or flat rep. By having FLAT == EXTERNAL + 1, we // can perform this check in a single branch as 'tag >= EXTERNAL' -// Likewise, we have some locations where we check for 'ring or external/flat', -// so likewise align RING to EXTERNAL. // Note that we can leave this optimization to the compiler. The compiler will // DTRT when it sees a condition like `tag == EXTERNAL || tag >= FLAT`. -static_assert(RING == BTREE + 1, "BTREE and RING not consecutive"); -static_assert(EXTERNAL == RING + 1, "BTREE and EXTERNAL not consecutive"); static_assert(FLAT == EXTERNAL + 1, "EXTERNAL and FLAT not consecutive"); struct CordRep { @@ -301,15 +278,12 @@ struct CordRep { // # LINT.ThenChange(cord_rep_btree.h:copy_raw) // Returns true if this instance's tag matches the requested type. - constexpr bool IsRing() const { return tag == RING; } constexpr bool IsSubstring() const { return tag == SUBSTRING; } constexpr bool IsCrc() const { return tag == CRC; } constexpr bool IsExternal() const { return tag == EXTERNAL; } constexpr bool IsFlat() const { return tag >= FLAT; } constexpr bool IsBtree() const { return tag == BTREE; } - inline CordRepRing* ring(); - inline const CordRepRing* ring() const; inline CordRepSubstring* substring(); inline const CordRepSubstring* substring() const; inline CordRepCrc* crc(); diff --git a/contrib/restricted/abseil-cpp/absl/strings/internal/cord_rep_ring.cc b/contrib/restricted/abseil-cpp/absl/strings/internal/cord_rep_ring.cc deleted file mode 100644 index af2fc7683d..0000000000 --- a/contrib/restricted/abseil-cpp/absl/strings/internal/cord_rep_ring.cc +++ /dev/null @@ -1,773 +0,0 @@ -// Copyright 2020 The Abseil Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include "absl/strings/internal/cord_rep_ring.h" - -#include <cassert> -#include <cstddef> -#include <cstdint> -#include <iostream> -#include <limits> -#include <memory> -#include <string> - -#include "absl/base/internal/raw_logging.h" -#include "absl/base/internal/throw_delegate.h" -#include "absl/base/macros.h" -#include "absl/container/inlined_vector.h" -#include "absl/strings/internal/cord_internal.h" -#include "absl/strings/internal/cord_rep_consume.h" -#include "absl/strings/internal/cord_rep_flat.h" - -namespace absl { -ABSL_NAMESPACE_BEGIN -namespace cord_internal { - -namespace { - -using index_type = CordRepRing::index_type; - -enum class Direction { kForward, kReversed }; - -inline bool IsFlatOrExternal(CordRep* rep) { - return rep->IsFlat() || rep->IsExternal(); -} - -// Verifies that n + extra <= kMaxCapacity: throws std::length_error otherwise. -inline void CheckCapacity(size_t n, size_t extra) { - if (ABSL_PREDICT_FALSE(extra > CordRepRing::kMaxCapacity - n)) { - base_internal::ThrowStdLengthError("Maximum capacity exceeded"); - } -} - -// Creates a flat from the provided string data, allocating up to `extra` -// capacity in the returned flat depending on kMaxFlatLength limitations. -// Requires `len` to be less or equal to `kMaxFlatLength` -CordRepFlat* CreateFlat(const char* s, size_t n, size_t extra = 0) { // NOLINT - assert(n <= kMaxFlatLength); - auto* rep = CordRepFlat::New(n + extra); - rep->length = n; - memcpy(rep->Data(), s, n); - return rep; -} - -// Unrefs the entries in `[head, tail)`. -// Requires all entries to be a FLAT or EXTERNAL node. -void UnrefEntries(const CordRepRing* rep, index_type head, index_type tail) { - rep->ForEach(head, tail, [rep](index_type ix) { - CordRep* child = rep->entry_child(ix); - if (!child->refcount.Decrement()) { - if (child->tag >= FLAT) { - CordRepFlat::Delete(child->flat()); - } else { - CordRepExternal::Delete(child->external()); - } - } - }); -} - -} // namespace - -std::ostream& operator<<(std::ostream& s, const CordRepRing& rep) { - // Note: 'pos' values are defined as size_t (for overflow reasons), but that - // prints really awkward for small prepended values such as -5. ssize_t is not - // portable (POSIX), so we use ptrdiff_t instead to cast to signed values. - s << " CordRepRing(" << &rep << ", length = " << rep.length - << ", head = " << rep.head_ << ", tail = " << rep.tail_ - << ", cap = " << rep.capacity_ << ", rc = " << rep.refcount.Get() - << ", begin_pos_ = " << static_cast<ptrdiff_t>(rep.begin_pos_) << ") {\n"; - CordRepRing::index_type head = rep.head(); - do { - CordRep* child = rep.entry_child(head); - s << " entry[" << head << "] length = " << rep.entry_length(head) - << ", child " << child << ", clen = " << child->length - << ", tag = " << static_cast<int>(child->tag) - << ", rc = " << child->refcount.Get() - << ", offset = " << rep.entry_data_offset(head) - << ", end_pos = " << static_cast<ptrdiff_t>(rep.entry_end_pos(head)) - << "\n"; - head = rep.advance(head); - } while (head != rep.tail()); - return s << "}\n"; -} - -void CordRepRing::AddDataOffset(index_type index, size_t n) { - entry_data_offset()[index] += static_cast<offset_type>(n); -} - -void CordRepRing::SubLength(index_type index, size_t n) { - entry_end_pos()[index] -= n; -} - -class CordRepRing::Filler { - public: - Filler(CordRepRing* rep, index_type pos) : rep_(rep), head_(pos), pos_(pos) {} - - index_type head() const { return head_; } - index_type pos() const { return pos_; } - - void Add(CordRep* child, size_t offset, pos_type end_pos) { - rep_->entry_end_pos()[pos_] = end_pos; - rep_->entry_child()[pos_] = child; - rep_->entry_data_offset()[pos_] = static_cast<offset_type>(offset); - pos_ = rep_->advance(pos_); - } - - private: - CordRepRing* rep_; - index_type head_; - index_type pos_; -}; - -#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL -constexpr size_t CordRepRing::kMaxCapacity; -#endif - -bool CordRepRing::IsValid(std::ostream& output) const { - if (capacity_ == 0) { - output << "capacity == 0"; - return false; - } - - if (head_ >= capacity_ || tail_ >= capacity_) { - output << "head " << head_ << " and/or tail " << tail_ << "exceed capacity " - << capacity_; - return false; - } - - const index_type back = retreat(tail_); - size_t pos_length = Distance(begin_pos_, entry_end_pos(back)); - if (pos_length != length) { - output << "length " << length << " does not match positional length " - << pos_length << " from begin_pos " << begin_pos_ << " and entry[" - << back << "].end_pos " << entry_end_pos(back); - return false; - } - - index_type head = head_; - pos_type begin_pos = begin_pos_; - do { - pos_type end_pos = entry_end_pos(head); - size_t entry_length = Distance(begin_pos, end_pos); - if (entry_length == 0) { - output << "entry[" << head << "] has an invalid length " << entry_length - << " from begin_pos " << begin_pos << " and end_pos " << end_pos; - return false; - } - - CordRep* child = entry_child(head); - if (child == nullptr) { - output << "entry[" << head << "].child == nullptr"; - return false; - } - if (child->tag < FLAT && child->tag != EXTERNAL) { - output << "entry[" << head << "].child has an invalid tag " - << static_cast<int>(child->tag); - return false; - } - - size_t offset = entry_data_offset(head); - if (offset >= child->length || entry_length > child->length - offset) { - output << "entry[" << head << "] has offset " << offset - << " and entry length " << entry_length - << " which are outside of the child's length of " << child->length; - return false; - } - - begin_pos = end_pos; - head = advance(head); - } while (head != tail_); - - return true; -} - -#ifdef EXTRA_CORD_RING_VALIDATION -CordRepRing* CordRepRing::Validate(CordRepRing* rep, const char* file, - int line) { - if (!rep->IsValid(std::cerr)) { - std::cerr << "\nERROR: CordRepRing corrupted"; - if (line) std::cerr << " at line " << line; - if (file) std::cerr << " in file " << file; - std::cerr << "\nContent = " << *rep; - abort(); - } - return rep; -} -#endif // EXTRA_CORD_RING_VALIDATION - -CordRepRing* CordRepRing::New(size_t capacity, size_t extra) { - CheckCapacity(capacity, extra); - - size_t size = AllocSize(capacity += extra); - void* mem = ::operator new(size); - auto* rep = new (mem) CordRepRing(static_cast<index_type>(capacity)); - rep->tag = RING; - rep->capacity_ = static_cast<index_type>(capacity); - rep->begin_pos_ = 0; - return rep; -} - -void CordRepRing::SetCapacityForTesting(size_t capacity) { - // Adjust for the changed layout - assert(capacity <= capacity_); - assert(head() == 0 || head() < tail()); - memmove(Layout::Partial(capacity).Pointer<1>(data_) + head(), - Layout::Partial(capacity_).Pointer<1>(data_) + head(), - entries() * sizeof(Layout::ElementType<1>)); - memmove(Layout::Partial(capacity, capacity).Pointer<2>(data_) + head(), - Layout::Partial(capacity_, capacity_).Pointer<2>(data_) + head(), - entries() * sizeof(Layout::ElementType<2>)); - capacity_ = static_cast<index_type>(capacity); -} - -void CordRepRing::Delete(CordRepRing* rep) { - assert(rep != nullptr && rep->IsRing()); -#if defined(__cpp_sized_deallocation) - size_t size = AllocSize(rep->capacity_); - rep->~CordRepRing(); - ::operator delete(rep, size); -#else - rep->~CordRepRing(); - ::operator delete(rep); -#endif -} - -void CordRepRing::Destroy(CordRepRing* rep) { - UnrefEntries(rep, rep->head(), rep->tail()); - Delete(rep); -} - -template <bool ref> -void CordRepRing::Fill(const CordRepRing* src, index_type head, - index_type tail) { - this->length = src->length; - head_ = 0; - tail_ = advance(0, src->entries(head, tail)); - begin_pos_ = src->begin_pos_; - - // TODO(mvels): there may be opportunities here for large buffers. - auto* dst_pos = entry_end_pos(); - auto* dst_child = entry_child(); - auto* dst_offset = entry_data_offset(); - src->ForEach(head, tail, [&](index_type index) { - *dst_pos++ = src->entry_end_pos(index); - CordRep* child = src->entry_child(index); - *dst_child++ = ref ? CordRep::Ref(child) : child; - *dst_offset++ = src->entry_data_offset(index); - }); -} - -CordRepRing* CordRepRing::Copy(CordRepRing* rep, index_type head, - index_type tail, size_t extra) { - CordRepRing* newrep = CordRepRing::New(rep->entries(head, tail), extra); - newrep->Fill<true>(rep, head, tail); - CordRep::Unref(rep); - return newrep; -} - -CordRepRing* CordRepRing::Mutable(CordRepRing* rep, size_t extra) { - // Get current number of entries, and check for max capacity. - size_t entries = rep->entries(); - - if (!rep->refcount.IsOne()) { - return Copy(rep, rep->head(), rep->tail(), extra); - } else if (entries + extra > rep->capacity()) { - const size_t min_grow = rep->capacity() + rep->capacity() / 2; - const size_t min_extra = (std::max)(extra, min_grow - entries); - CordRepRing* newrep = CordRepRing::New(entries, min_extra); - newrep->Fill<false>(rep, rep->head(), rep->tail()); - CordRepRing::Delete(rep); - return newrep; - } else { - return rep; - } -} - -Span<char> CordRepRing::GetAppendBuffer(size_t size) { - assert(refcount.IsOne()); - index_type back = retreat(tail_); - CordRep* child = entry_child(back); - if (child->tag >= FLAT && child->refcount.IsOne()) { - size_t capacity = child->flat()->Capacity(); - pos_type end_pos = entry_end_pos(back); - size_t data_offset = entry_data_offset(back); - size_t entry_length = Distance(entry_begin_pos(back), end_pos); - size_t used = data_offset + entry_length; - if (size_t n = (std::min)(capacity - used, size)) { - child->length = data_offset + entry_length + n; - entry_end_pos()[back] = end_pos + n; - this->length += n; - return {child->flat()->Data() + used, n}; - } - } - return {nullptr, 0}; -} - -Span<char> CordRepRing::GetPrependBuffer(size_t size) { - assert(refcount.IsOne()); - CordRep* child = entry_child(head_); - size_t data_offset = entry_data_offset(head_); - if (data_offset && child->refcount.IsOne() && child->tag >= FLAT) { - size_t n = (std::min)(data_offset, size); - this->length += n; - begin_pos_ -= n; - data_offset -= n; - entry_data_offset()[head_] = static_cast<offset_type>(data_offset); - return {child->flat()->Data() + data_offset, n}; - } - return {nullptr, 0}; -} - -CordRepRing* CordRepRing::CreateFromLeaf(CordRep* child, size_t offset, - size_t len, size_t extra) { - CordRepRing* rep = CordRepRing::New(1, extra); - rep->head_ = 0; - rep->tail_ = rep->advance(0); - rep->length = len; - rep->entry_end_pos()[0] = len; - rep->entry_child()[0] = child; - rep->entry_data_offset()[0] = static_cast<offset_type>(offset); - return Validate(rep); -} - -CordRepRing* CordRepRing::CreateSlow(CordRep* child, size_t extra) { - CordRepRing* rep = nullptr; - Consume(child, [&](CordRep* child_arg, size_t offset, size_t len) { - if (IsFlatOrExternal(child_arg)) { - rep = rep ? AppendLeaf(rep, child_arg, offset, len) - : CreateFromLeaf(child_arg, offset, len, extra); - } else if (rep) { - rep = AddRing<AddMode::kAppend>(rep, child_arg->ring(), offset, len); - } else if (offset == 0 && child_arg->length == len) { - rep = Mutable(child_arg->ring(), extra); - } else { - rep = SubRing(child_arg->ring(), offset, len, extra); - } - }); - return Validate(rep, nullptr, __LINE__); -} - -CordRepRing* CordRepRing::Create(CordRep* child, size_t extra) { - size_t length = child->length; - if (IsFlatOrExternal(child)) { - return CreateFromLeaf(child, 0, length, extra); - } - if (child->IsRing()) { - return Mutable(child->ring(), extra); - } - return CreateSlow(child, extra); -} - -template <CordRepRing::AddMode mode> -CordRepRing* CordRepRing::AddRing(CordRepRing* rep, CordRepRing* ring, - size_t offset, size_t len) { - assert(offset < ring->length); - constexpr bool append = mode == AddMode::kAppend; - Position head = ring->Find(offset); - Position tail = ring->FindTail(head.index, offset + len); - const index_type entries = ring->entries(head.index, tail.index); - - rep = Mutable(rep, entries); - - // The delta for making ring[head].end_pos into 'len - offset' - const pos_type delta_length = - (append ? rep->begin_pos_ + rep->length : rep->begin_pos_ - len) - - ring->entry_begin_pos(head.index) - head.offset; - - // Start filling at `tail`, or `entries` before `head` - Filler filler(rep, append ? rep->tail_ : rep->retreat(rep->head_, entries)); - - if (ring->refcount.IsOne()) { - // Copy entries from source stealing the ref and adjusting the end position. - // Commit the filler as this is no-op. - ring->ForEach(head.index, tail.index, [&](index_type ix) { - filler.Add(ring->entry_child(ix), ring->entry_data_offset(ix), - ring->entry_end_pos(ix) + delta_length); - }); - - // Unref entries we did not copy over, and delete source. - if (head.index != ring->head_) UnrefEntries(ring, ring->head_, head.index); - if (tail.index != ring->tail_) UnrefEntries(ring, tail.index, ring->tail_); - CordRepRing::Delete(ring); - } else { - ring->ForEach(head.index, tail.index, [&](index_type ix) { - CordRep* child = ring->entry_child(ix); - filler.Add(child, ring->entry_data_offset(ix), - ring->entry_end_pos(ix) + delta_length); - CordRep::Ref(child); - }); - CordRepRing::Unref(ring); - } - - if (head.offset) { - // Increase offset of first 'source' entry appended or prepended. - // This is always the entry in `filler.head()` - rep->AddDataOffset(filler.head(), head.offset); - } - - if (tail.offset) { - // Reduce length of last 'source' entry appended or prepended. - // This is always the entry tailed by `filler.pos()` - rep->SubLength(rep->retreat(filler.pos()), tail.offset); - } - - // Commit changes - rep->length += len; - if (append) { - rep->tail_ = filler.pos(); - } else { - rep->head_ = filler.head(); - rep->begin_pos_ -= len; - } - - return Validate(rep); -} - -CordRepRing* CordRepRing::AppendSlow(CordRepRing* rep, CordRep* child) { - Consume(child, [&rep](CordRep* child_arg, size_t offset, size_t len) { - if (child_arg->IsRing()) { - rep = AddRing<AddMode::kAppend>(rep, child_arg->ring(), offset, len); - } else { - rep = AppendLeaf(rep, child_arg, offset, len); - } - }); - return rep; -} - -CordRepRing* CordRepRing::AppendLeaf(CordRepRing* rep, CordRep* child, - size_t offset, size_t len) { - rep = Mutable(rep, 1); - index_type back = rep->tail_; - const pos_type begin_pos = rep->begin_pos_ + rep->length; - rep->tail_ = rep->advance(rep->tail_); - rep->length += len; - rep->entry_end_pos()[back] = begin_pos + len; - rep->entry_child()[back] = child; - rep->entry_data_offset()[back] = static_cast<offset_type>(offset); - return Validate(rep, nullptr, __LINE__); -} - -CordRepRing* CordRepRing::Append(CordRepRing* rep, CordRep* child) { - size_t length = child->length; - if (IsFlatOrExternal(child)) { - return AppendLeaf(rep, child, 0, length); - } - if (child->IsRing()) { - return AddRing<AddMode::kAppend>(rep, child->ring(), 0, length); - } - return AppendSlow(rep, child); -} - -CordRepRing* CordRepRing::PrependSlow(CordRepRing* rep, CordRep* child) { - ReverseConsume(child, [&](CordRep* child_arg, size_t offset, size_t len) { - if (IsFlatOrExternal(child_arg)) { - rep = PrependLeaf(rep, child_arg, offset, len); - } else { - rep = AddRing<AddMode::kPrepend>(rep, child_arg->ring(), offset, len); - } - }); - return Validate(rep); -} - -CordRepRing* CordRepRing::PrependLeaf(CordRepRing* rep, CordRep* child, - size_t offset, size_t len) { - rep = Mutable(rep, 1); - index_type head = rep->retreat(rep->head_); - pos_type end_pos = rep->begin_pos_; - rep->head_ = head; - rep->length += len; - rep->begin_pos_ -= len; - rep->entry_end_pos()[head] = end_pos; - rep->entry_child()[head] = child; - rep->entry_data_offset()[head] = static_cast<offset_type>(offset); - return Validate(rep); -} - -CordRepRing* CordRepRing::Prepend(CordRepRing* rep, CordRep* child) { - size_t length = child->length; - if (IsFlatOrExternal(child)) { - return PrependLeaf(rep, child, 0, length); - } - if (child->IsRing()) { - return AddRing<AddMode::kPrepend>(rep, child->ring(), 0, length); - } - return PrependSlow(rep, child); -} - -CordRepRing* CordRepRing::Append(CordRepRing* rep, absl::string_view data, - size_t extra) { - if (rep->refcount.IsOne()) { - Span<char> avail = rep->GetAppendBuffer(data.length()); - if (!avail.empty()) { - memcpy(avail.data(), data.data(), avail.length()); - data.remove_prefix(avail.length()); - } - } - if (data.empty()) return Validate(rep); - - const size_t flats = (data.length() - 1) / kMaxFlatLength + 1; - rep = Mutable(rep, flats); - - Filler filler(rep, rep->tail_); - pos_type pos = rep->begin_pos_ + rep->length; - - while (data.length() >= kMaxFlatLength) { - auto* flat = CreateFlat(data.data(), kMaxFlatLength); - filler.Add(flat, 0, pos += kMaxFlatLength); - data.remove_prefix(kMaxFlatLength); - } - - if (data.length()) { - auto* flat = CreateFlat(data.data(), data.length(), extra); - filler.Add(flat, 0, pos += data.length()); - } - - rep->length = pos - rep->begin_pos_; - rep->tail_ = filler.pos(); - - return Validate(rep); -} - -CordRepRing* CordRepRing::Prepend(CordRepRing* rep, absl::string_view data, - size_t extra) { - if (rep->refcount.IsOne()) { - Span<char> avail = rep->GetPrependBuffer(data.length()); - if (!avail.empty()) { - const char* tail = data.data() + data.length() - avail.length(); - memcpy(avail.data(), tail, avail.length()); - data.remove_suffix(avail.length()); - } - } - if (data.empty()) return rep; - - const size_t flats = (data.length() - 1) / kMaxFlatLength + 1; - rep = Mutable(rep, flats); - pos_type pos = rep->begin_pos_; - Filler filler(rep, rep->retreat(rep->head_, static_cast<index_type>(flats))); - - size_t first_size = data.size() - (flats - 1) * kMaxFlatLength; - CordRepFlat* flat = CordRepFlat::New(first_size + extra); - flat->length = first_size + extra; - memcpy(flat->Data() + extra, data.data(), first_size); - data.remove_prefix(first_size); - filler.Add(flat, extra, pos); - pos -= first_size; - - while (!data.empty()) { - assert(data.size() >= kMaxFlatLength); - flat = CreateFlat(data.data(), kMaxFlatLength); - filler.Add(flat, 0, pos); - pos -= kMaxFlatLength; - data.remove_prefix(kMaxFlatLength); - } - - rep->head_ = filler.head(); - rep->length += rep->begin_pos_ - pos; - rep->begin_pos_ = pos; - - return Validate(rep); -} - -// 32 entries is 32 * sizeof(pos_type) = 4 cache lines on x86 -static constexpr index_type kBinarySearchThreshold = 32; -static constexpr index_type kBinarySearchEndCount = 8; - -template <bool wrap> -CordRepRing::index_type CordRepRing::FindBinary(index_type head, - index_type tail, - size_t offset) const { - index_type count = tail + (wrap ? capacity_ : 0) - head; - do { - count = (count - 1) / 2; - assert(count < entries(head, tail_)); - index_type mid = wrap ? advance(head, count) : head + count; - index_type after_mid = wrap ? advance(mid) : mid + 1; - bool larger = (offset >= entry_end_offset(mid)); - head = larger ? after_mid : head; - tail = larger ? tail : mid; - assert(head != tail); - } while (ABSL_PREDICT_TRUE(count > kBinarySearchEndCount)); - return head; -} - -CordRepRing::Position CordRepRing::FindSlow(index_type head, - size_t offset) const { - index_type tail = tail_; - - // Binary search until we are good for linear search - // Optimize for branchless / non wrapping ops - if (tail > head) { - index_type count = tail - head; - if (count > kBinarySearchThreshold) { - head = FindBinary<false>(head, tail, offset); - } - } else { - index_type count = capacity_ + tail - head; - if (count > kBinarySearchThreshold) { - head = FindBinary<true>(head, tail, offset); - } - } - - pos_type pos = entry_begin_pos(head); - pos_type end_pos = entry_end_pos(head); - while (offset >= Distance(begin_pos_, end_pos)) { - head = advance(head); - pos = end_pos; - end_pos = entry_end_pos(head); - } - - return {head, offset - Distance(begin_pos_, pos)}; -} - -CordRepRing::Position CordRepRing::FindTailSlow(index_type head, - size_t offset) const { - index_type tail = tail_; - const size_t tail_offset = offset - 1; - - // Binary search until we are good for linear search - // Optimize for branchless / non wrapping ops - if (tail > head) { - index_type count = tail - head; - if (count > kBinarySearchThreshold) { - head = FindBinary<false>(head, tail, tail_offset); - } - } else { - index_type count = capacity_ + tail - head; - if (count > kBinarySearchThreshold) { - head = FindBinary<true>(head, tail, tail_offset); - } - } - - size_t end_offset = entry_end_offset(head); - while (tail_offset >= end_offset) { - head = advance(head); - end_offset = entry_end_offset(head); - } - - return {advance(head), end_offset - offset}; -} - -char CordRepRing::GetCharacter(size_t offset) const { - assert(offset < length); - - Position pos = Find(offset); - size_t data_offset = entry_data_offset(pos.index) + pos.offset; - return GetRepData(entry_child(pos.index))[data_offset]; -} - -CordRepRing* CordRepRing::SubRing(CordRepRing* rep, size_t offset, - size_t len, size_t extra) { - assert(offset <= rep->length); - assert(offset <= rep->length - len); - - if (len == 0) { - CordRep::Unref(rep); - return nullptr; - } - - // Find position of first byte - Position head = rep->Find(offset); - Position tail = rep->FindTail(head.index, offset + len); - const size_t new_entries = rep->entries(head.index, tail.index); - - if (rep->refcount.IsOne() && extra <= (rep->capacity() - new_entries)) { - // We adopt a privately owned rep and no extra entries needed. - if (head.index != rep->head_) UnrefEntries(rep, rep->head_, head.index); - if (tail.index != rep->tail_) UnrefEntries(rep, tail.index, rep->tail_); - rep->head_ = head.index; - rep->tail_ = tail.index; - } else { - // Copy subset to new rep - rep = Copy(rep, head.index, tail.index, extra); - head.index = rep->head_; - tail.index = rep->tail_; - } - - // Adjust begin_pos and length - rep->length = len; - rep->begin_pos_ += offset; - - // Adjust head and tail blocks - if (head.offset) { - rep->AddDataOffset(head.index, head.offset); - } - if (tail.offset) { - rep->SubLength(rep->retreat(tail.index), tail.offset); - } - - return Validate(rep); -} - -CordRepRing* CordRepRing::RemovePrefix(CordRepRing* rep, size_t len, - size_t extra) { - assert(len <= rep->length); - if (len == rep->length) { - CordRep::Unref(rep); - return nullptr; - } - - Position head = rep->Find(len); - if (rep->refcount.IsOne()) { - if (head.index != rep->head_) UnrefEntries(rep, rep->head_, head.index); - rep->head_ = head.index; - } else { - rep = Copy(rep, head.index, rep->tail_, extra); - head.index = rep->head_; - } - - // Adjust begin_pos and length - rep->length -= len; - rep->begin_pos_ += len; - - // Adjust head block - if (head.offset) { - rep->AddDataOffset(head.index, head.offset); - } - - return Validate(rep); -} - -CordRepRing* CordRepRing::RemoveSuffix(CordRepRing* rep, size_t len, - size_t extra) { - assert(len <= rep->length); - - if (len == rep->length) { - CordRep::Unref(rep); - return nullptr; - } - - Position tail = rep->FindTail(rep->length - len); - if (rep->refcount.IsOne()) { - // We adopt a privately owned rep, scrub. - if (tail.index != rep->tail_) UnrefEntries(rep, tail.index, rep->tail_); - rep->tail_ = tail.index; - } else { - // Copy subset to new rep - rep = Copy(rep, rep->head_, tail.index, extra); - tail.index = rep->tail_; - } - - // Adjust length - rep->length -= len; - - // Adjust tail block - if (tail.offset) { - rep->SubLength(rep->retreat(tail.index), tail.offset); - } - - return Validate(rep); -} - -} // namespace cord_internal -ABSL_NAMESPACE_END -} // namespace absl diff --git a/contrib/restricted/abseil-cpp/absl/strings/internal/cord_rep_ring.h b/contrib/restricted/abseil-cpp/absl/strings/internal/cord_rep_ring.h deleted file mode 100644 index 79a2fdb1c3..0000000000 --- a/contrib/restricted/abseil-cpp/absl/strings/internal/cord_rep_ring.h +++ /dev/null @@ -1,607 +0,0 @@ -// Copyright 2020 The Abseil Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef ABSL_STRINGS_INTERNAL_CORD_REP_RING_H_ -#define ABSL_STRINGS_INTERNAL_CORD_REP_RING_H_ - -#include <cassert> -#include <cstddef> -#include <cstdint> -#include <iosfwd> -#include <limits> -#include <memory> - -#include "absl/container/internal/layout.h" -#include "absl/strings/internal/cord_internal.h" -#include "absl/strings/internal/cord_rep_flat.h" - -namespace absl { -ABSL_NAMESPACE_BEGIN -namespace cord_internal { - -// All operations modifying a ring buffer are implemented as static methods -// requiring a CordRepRing instance with a reference adopted by the method. -// -// The methods return the modified ring buffer, which may be equal to the input -// if the input was not shared, and having large enough capacity to accommodate -// any newly added node(s). Otherwise, a copy of the input rep with the new -// node(s) added is returned. -// -// Any modification on non shared ring buffers with enough capacity will then -// require minimum atomic operations. Caller should where possible provide -// reasonable `extra` hints for both anticipated extra `flat` byte space, as -// well as anticipated extra nodes required for complex operations. -// -// Example of code creating a ring buffer, adding some data to it, -// and discarding the buffer when done: -// -// void FunWithRings() { -// // Create ring with 3 flats -// CordRep* flat = CreateFlat("Hello"); -// CordRepRing* ring = CordRepRing::Create(flat, 2); -// ring = CordRepRing::Append(ring, CreateFlat(" ")); -// ring = CordRepRing::Append(ring, CreateFlat("world")); -// DoSomethingWithRing(ring); -// CordRep::Unref(ring); -// } -// -// Example of code Copying an existing ring buffer and modifying it: -// -// void MoreFunWithRings(CordRepRing* src) { -// CordRepRing* ring = CordRep::Ref(src)->ring(); -// ring = CordRepRing::Append(ring, CreateFlat("Hello")); -// ring = CordRepRing::Append(ring, CreateFlat(" ")); -// ring = CordRepRing::Append(ring, CreateFlat("world")); -// DoSomethingWithRing(ring); -// CordRep::Unref(ring); -// } -// -class CordRepRing : public CordRep { - public: - // `pos_type` represents a 'logical position'. A CordRepRing instance has a - // `begin_pos` (default 0), and each node inside the buffer will have an - // `end_pos` which is the `end_pos` of the previous node (or `begin_pos`) plus - // this node's length. The purpose is to allow for a binary search on this - // position, while allowing O(1) prepend and append operations. - using pos_type = size_t; - - // `index_type` is the type for the `head`, `tail` and `capacity` indexes. - // Ring buffers are limited to having no more than four billion entries. - using index_type = uint32_t; - - // `offset_type` is the type for the data offset inside a child rep's data. - using offset_type = uint32_t; - - // Position holds the node index and relative offset into the node for - // some physical offset in the contained data as returned by the Find() - // and FindTail() methods. - struct Position { - index_type index; - size_t offset; - }; - - // The maximum # of child nodes that can be hosted inside a CordRepRing. - static constexpr size_t kMaxCapacity = (std::numeric_limits<uint32_t>::max)(); - - // CordRepring can not be default constructed, moved, copied or assigned. - CordRepRing() = delete; - CordRepRing(const CordRepRing&) = delete; - CordRepRing& operator=(const CordRepRing&) = delete; - - // Returns true if this instance is valid, false if some or all of the - // invariants are broken. Intended for debug purposes only. - // `output` receives an explanation of the broken invariants. - bool IsValid(std::ostream& output) const; - - // Returns the size in bytes for a CordRepRing with `capacity' entries. - static constexpr size_t AllocSize(size_t capacity); - - // Returns the distance in bytes from `pos` to `end_pos`. - static constexpr size_t Distance(pos_type pos, pos_type end_pos); - - // Creates a new ring buffer from the provided `rep`. Adopts a reference - // on `rep`. The returned ring buffer has a capacity of at least `extra + 1` - static CordRepRing* Create(CordRep* child, size_t extra = 0); - - // `head`, `tail` and `capacity` indexes defining the ring buffer boundaries. - index_type head() const { return head_; } - index_type tail() const { return tail_; } - index_type capacity() const { return capacity_; } - - // Returns the number of entries in this instance. - index_type entries() const { return entries(head_, tail_); } - - // Returns the logical begin position of this instance. - pos_type begin_pos() const { return begin_pos_; } - - // Returns the number of entries for a given head-tail range. - // Requires `head` and `tail` values to be less than `capacity()`. - index_type entries(index_type head, index_type tail) const { - assert(head < capacity_ && tail < capacity_); - return tail - head + ((tail > head) ? 0 : capacity_); - } - - // Returns the logical end position of entry `index`. - pos_type const& entry_end_pos(index_type index) const { - assert(IsValidIndex(index)); - return Layout::Partial().Pointer<0>(data_)[index]; - } - - // Returns the child pointer of entry `index`. - CordRep* const& entry_child(index_type index) const { - assert(IsValidIndex(index)); - return Layout::Partial(capacity()).Pointer<1>(data_)[index]; - } - - // Returns the data offset of entry `index` - offset_type const& entry_data_offset(index_type index) const { - assert(IsValidIndex(index)); - return Layout::Partial(capacity(), capacity()).Pointer<2>(data_)[index]; - } - - // Appends the provided child node to the `rep` instance. - // Adopts a reference from `rep` and `child` which may not be null. - // If the provided child is a FLAT or EXTERNAL node, or a SUBSTRING node - // containing a FLAT or EXTERNAL node, then flat or external the node is added - // 'as is', with an offset added for the SUBSTRING case. - // If the provided child is a RING or CONCAT tree, or a SUBSTRING of a RING or - // CONCAT tree, then all child nodes not excluded by any start offset or - // length values are added recursively. - static CordRepRing* Append(CordRepRing* rep, CordRep* child); - - // Appends the provided string data to the `rep` instance. - // This function will attempt to utilize any remaining capacity in the last - // node of the input if that node is not shared (directly or indirectly), and - // of type FLAT. Remaining data will be added as one or more FLAT nodes. - // Any last node added to the ring buffer will be allocated with up to - // `extra` bytes of capacity for (anticipated) subsequent append actions. - static CordRepRing* Append(CordRepRing* rep, string_view data, - size_t extra = 0); - - // Prepends the provided child node to the `rep` instance. - // Adopts a reference from `rep` and `child` which may not be null. - // If the provided child is a FLAT or EXTERNAL node, or a SUBSTRING node - // containing a FLAT or EXTERNAL node, then flat or external the node is - // prepended 'as is', with an optional offset added for the SUBSTRING case. - // If the provided child is a RING or CONCAT tree, or a SUBSTRING of a RING - // or CONCAT tree, then all child nodes not excluded by any start offset or - // length values are added recursively. - static CordRepRing* Prepend(CordRepRing* rep, CordRep* child); - - // Prepends the provided string data to the `rep` instance. - // This function will attempt to utilize any remaining capacity in the first - // node of the input if that node is not shared (directly or indirectly), and - // of type FLAT. Remaining data will be added as one or more FLAT nodes. - // Any first node prepnded to the ring buffer will be allocated with up to - // `extra` bytes of capacity for (anticipated) subsequent prepend actions. - static CordRepRing* Prepend(CordRepRing* rep, string_view data, - size_t extra = 0); - - // Returns a span referencing potentially unused capacity in the last node. - // The returned span may be empty if no such capacity is available, or if the - // current instance is shared. Else, a span of size `n <= size` is returned. - // If non empty, the ring buffer is adjusted to the new length, with the newly - // added capacity left uninitialized. Callers should assign a value to the - // entire span before any other operations on this instance. - Span<char> GetAppendBuffer(size_t size); - - // Returns a span referencing potentially unused capacity in the first node. - // This function is identical to GetAppendBuffer except that it returns a span - // referencing up to `size` capacity directly before the existing data. - Span<char> GetPrependBuffer(size_t size); - - // Returns a cord ring buffer containing `len` bytes of data starting at - // `offset`. If the input is not shared, this function will remove all head - // and tail child nodes outside of the requested range, and adjust the new - // head and tail nodes as required. If the input is shared, this function - // returns a new instance sharing some or all of the nodes from the input. - static CordRepRing* SubRing(CordRepRing* r, size_t offset, size_t len, - size_t extra = 0); - - // Returns a cord ring buffer with the first `len` bytes removed. - // If the input is not shared, this function will remove all head child nodes - // fully inside the first `length` bytes, and adjust the new head as required. - // If the input is shared, this function returns a new instance sharing some - // or all of the nodes from the input. - static CordRepRing* RemoveSuffix(CordRepRing* r, size_t len, - size_t extra = 0); - - // Returns a cord ring buffer with the last `len` bytes removed. - // If the input is not shared, this function will remove all head child nodes - // fully inside the first `length` bytes, and adjust the new head as required. - // If the input is shared, this function returns a new instance sharing some - // or all of the nodes from the input. - static CordRepRing* RemovePrefix(CordRepRing* r, size_t len, - size_t extra = 0); - - // Returns the character at `offset`. Requires that `offset < length`. - char GetCharacter(size_t offset) const; - - // Returns true if this instance manages a single contiguous buffer, in which - // case the (optional) output parameter `fragment` is set. Otherwise, the - // function returns false, and `fragment` is left unchanged. - bool IsFlat(absl::string_view* fragment) const; - - // Returns true if the data starting at `offset` with length `len` is - // managed by this instance inside a single contiguous buffer, in which case - // the (optional) output parameter `fragment` is set to the contiguous memory - // starting at offset `offset` with length `length`. Otherwise, the function - // returns false, and `fragment` is left unchanged. - bool IsFlat(size_t offset, size_t len, absl::string_view* fragment) const; - - // Testing only: set capacity to requested capacity. - void SetCapacityForTesting(size_t capacity); - - // Returns the CordRep data pointer for the provided CordRep. - // Requires that the provided `rep` is either a FLAT or EXTERNAL CordRep. - static const char* GetLeafData(const CordRep* rep); - - // Returns the CordRep data pointer for the provided CordRep. - // Requires that `rep` is either a FLAT, EXTERNAL, or SUBSTRING CordRep. - static const char* GetRepData(const CordRep* rep); - - // Advances the provided position, wrapping around capacity as needed. - // Requires `index` < capacity() - inline index_type advance(index_type index) const; - - // Advances the provided position by 'n`, wrapping around capacity as needed. - // Requires `index` < capacity() and `n` <= capacity. - inline index_type advance(index_type index, index_type n) const; - - // Retreats the provided position, wrapping around 0 as needed. - // Requires `index` < capacity() - inline index_type retreat(index_type index) const; - - // Retreats the provided position by 'n', wrapping around 0 as needed. - // Requires `index` < capacity() - inline index_type retreat(index_type index, index_type n) const; - - // Returns the logical begin position of entry `index` - pos_type const& entry_begin_pos(index_type index) const { - return (index == head_) ? begin_pos_ : entry_end_pos(retreat(index)); - } - - // Returns the physical start offset of entry `index` - size_t entry_start_offset(index_type index) const { - return Distance(begin_pos_, entry_begin_pos(index)); - } - - // Returns the physical end offset of entry `index` - size_t entry_end_offset(index_type index) const { - return Distance(begin_pos_, entry_end_pos(index)); - } - - // Returns the data length for entry `index` - size_t entry_length(index_type index) const { - return Distance(entry_begin_pos(index), entry_end_pos(index)); - } - - // Returns the data for entry `index` - absl::string_view entry_data(index_type index) const; - - // Returns the position for `offset` as {index, prefix}. `index` holds the - // index of the entry at the specified offset and `prefix` holds the relative - // offset inside that entry. - // Requires `offset` < length. - // - // For example we can implement GetCharacter(offset) as: - // char GetCharacter(size_t offset) { - // Position pos = this->Find(offset); - // return this->entry_data(pos.pos)[pos.offset]; - // } - inline Position Find(size_t offset) const; - - // Find starting at `head` - inline Position Find(index_type head, size_t offset) const; - - // Returns the tail position for `offset` as {tail index, suffix}. - // `tail index` holds holds the index of the entry holding the offset directly - // before 'offset` advanced by one. 'suffix` holds the relative offset from - // that relative offset in the entry to the end of the entry. - // For example, FindTail(length) will return {tail(), 0}, FindTail(length - 5) - // will return {retreat(tail), 5)} provided the preceding entry contains at - // least 5 bytes of data. - // Requires offset >= 1 && offset <= length. - // - // This function is very useful in functions that need to clip the end of some - // ring buffer such as 'RemovePrefix'. - // For example, we could implement RemovePrefix for non shared instances as: - // void RemoveSuffix(size_t n) { - // Position pos = FindTail(length - n); - // UnrefEntries(pos.pos, this->tail_); - // this->tail_ = pos.pos; - // entry(retreat(pos.pos)).end_pos -= pos.offset; - // } - inline Position FindTail(size_t offset) const; - - // Find tail starting at `head` - inline Position FindTail(index_type head, size_t offset) const; - - // Invokes f(index_type index) for each entry inside the range [head, tail> - template <typename F> - void ForEach(index_type head, index_type tail, F&& f) const { - index_type n1 = (tail > head) ? tail : capacity_; - for (index_type i = head; i < n1; ++i) f(i); - if (tail <= head) { - for (index_type i = 0; i < tail; ++i) f(i); - } - } - - // Invokes f(index_type index) for each entry inside this instance. - template <typename F> - void ForEach(F&& f) const { - ForEach(head_, tail_, std::forward<F>(f)); - } - - // Dump this instance's data tp stream `s` in human readable format, excluding - // the actual data content itself. Intended for debug purposes only. - friend std::ostream& operator<<(std::ostream& s, const CordRepRing& rep); - - private: - enum class AddMode { kAppend, kPrepend }; - - using Layout = container_internal::Layout<pos_type, CordRep*, offset_type>; - - class Filler; - class Transaction; - class CreateTransaction; - - static constexpr size_t kLayoutAlignment = Layout::Partial().Alignment(); - - // Creates a new CordRepRing. - explicit CordRepRing(index_type capacity) : capacity_(capacity) {} - - // Returns true if `index` is a valid index into this instance. - bool IsValidIndex(index_type index) const; - - // Debug use only: validates the provided CordRepRing invariants. - // Verification of all CordRepRing methods can be enabled by defining - // EXTRA_CORD_RING_VALIDATION, i.e.: `--copts=-DEXTRA_CORD_RING_VALIDATION` - // Verification is VERY expensive, so only do it for debugging purposes. - static CordRepRing* Validate(CordRepRing* rep, const char* file = nullptr, - int line = 0); - - // Allocates a CordRepRing large enough to hold `capacity + extra' entries. - // The returned capacity may be larger if the allocated memory allows for it. - // The maximum capacity of a CordRepRing is capped at kMaxCapacity. - // Throws `std::length_error` if `capacity + extra' exceeds kMaxCapacity. - static CordRepRing* New(size_t capacity, size_t extra); - - // Deallocates (but does not destroy) the provided ring buffer. - static void Delete(CordRepRing* rep); - - // Destroys the provided ring buffer, decrementing the reference count of all - // contained child CordReps. The provided 1\`rep` should have a ref count of - // one (pre decrement destroy call observing `refcount.IsOne()`) or zero - // (post decrement destroy call observing `!refcount.Decrement()`). - static void Destroy(CordRepRing* rep); - - // Returns a mutable reference to the logical end position array. - pos_type* entry_end_pos() { - return Layout::Partial().Pointer<0>(data_); - } - - // Returns a mutable reference to the child pointer array. - CordRep** entry_child() { - return Layout::Partial(capacity()).Pointer<1>(data_); - } - - // Returns a mutable reference to the data offset array. - offset_type* entry_data_offset() { - return Layout::Partial(capacity(), capacity()).Pointer<2>(data_); - } - - // Find implementations for the non fast path 0 / length cases. - Position FindSlow(index_type head, size_t offset) const; - Position FindTailSlow(index_type head, size_t offset) const; - - // Finds the index of the first node that is inside a reasonable distance - // of the node at `offset` from which we can continue with a linear search. - template <bool wrap> - index_type FindBinary(index_type head, index_type tail, size_t offset) const; - - // Fills the current (initialized) instance from the provided source, copying - // entries [head, tail). Adds a reference to copied entries if `ref` is true. - template <bool ref> - void Fill(const CordRepRing* src, index_type head, index_type tail); - - // Create a copy of 'rep', copying all entries [head, tail), allocating room - // for `extra` entries. Adds a reference on all copied entries. - static CordRepRing* Copy(CordRepRing* rep, index_type head, index_type tail, - size_t extra = 0); - - // Returns a Mutable CordRepRing reference from `rep` with room for at least - // `extra` additional nodes. Adopts a reference count from `rep`. - // This function will return `rep` if, and only if: - // - rep.entries + extra <= rep.capacity - // - rep.refcount == 1 - // Otherwise, this function will create a new copy of `rep` with additional - // capacity to satisfy `extra` extra nodes, and unref the old `rep` instance. - // - // If a new CordRepRing can not be allocated, or the new capacity would exceed - // the maximum capacity, then the input is consumed only, and an exception is - // thrown. - static CordRepRing* Mutable(CordRepRing* rep, size_t extra); - - // Slow path for Append(CordRepRing* rep, CordRep* child). This function is - // exercised if the provided `child` in Append() is not a leaf node, i.e., a - // ring buffer or old (concat) cord tree. - static CordRepRing* AppendSlow(CordRepRing* rep, CordRep* child); - - // Appends the provided leaf node. Requires `child` to be FLAT or EXTERNAL. - static CordRepRing* AppendLeaf(CordRepRing* rep, CordRep* child, - size_t offset, size_t length); - - // Prepends the provided leaf node. Requires `child` to be FLAT or EXTERNAL. - static CordRepRing* PrependLeaf(CordRepRing* rep, CordRep* child, - size_t offset, size_t length); - - // Slow path for Prepend(CordRepRing* rep, CordRep* child). This function is - // exercised if the provided `child` in Prepend() is not a leaf node, i.e., a - // ring buffer or old (concat) cord tree. - static CordRepRing* PrependSlow(CordRepRing* rep, CordRep* child); - - // Slow path for Create(CordRep* child, size_t extra). This function is - // exercised if the provided `child` in Prepend() is not a leaf node, i.e., a - // ring buffer or old (concat) cord tree. - static CordRepRing* CreateSlow(CordRep* child, size_t extra); - - // Creates a new ring buffer from the provided `child` leaf node. Requires - // `child` to be FLAT or EXTERNAL. on `rep`. - // The returned ring buffer has a capacity of at least `1 + extra` - static CordRepRing* CreateFromLeaf(CordRep* child, size_t offset, - size_t length, size_t extra); - - // Appends or prepends (depending on AddMode) the ring buffer in `ring' to - // `rep` starting at `offset` with length `len`. - template <AddMode mode> - static CordRepRing* AddRing(CordRepRing* rep, CordRepRing* ring, - size_t offset, size_t len); - - // Increases the data offset for entry `index` by `n`. - void AddDataOffset(index_type index, size_t n); - - // Decreases the length for entry `index` by `n`. - void SubLength(index_type index, size_t n); - - index_type head_; - index_type tail_; - index_type capacity_; - pos_type begin_pos_; - - alignas(kLayoutAlignment) char data_[kLayoutAlignment]; - - friend struct CordRep; -}; - -constexpr size_t CordRepRing::AllocSize(size_t capacity) { - return sizeof(CordRepRing) - sizeof(data_) + - Layout(capacity, capacity, capacity).AllocSize(); -} - -inline constexpr size_t CordRepRing::Distance(pos_type pos, pos_type end_pos) { - return (end_pos - pos); -} - -inline const char* CordRepRing::GetLeafData(const CordRep* rep) { - return rep->tag != EXTERNAL ? rep->flat()->Data() : rep->external()->base; -} - -inline const char* CordRepRing::GetRepData(const CordRep* rep) { - if (rep->tag >= FLAT) return rep->flat()->Data(); - if (rep->tag == EXTERNAL) return rep->external()->base; - return GetLeafData(rep->substring()->child) + rep->substring()->start; -} - -inline CordRepRing::index_type CordRepRing::advance(index_type index) const { - assert(index < capacity_); - return ++index == capacity_ ? 0 : index; -} - -inline CordRepRing::index_type CordRepRing::advance(index_type index, - index_type n) const { - assert(index < capacity_ && n <= capacity_); - return (index += n) >= capacity_ ? index - capacity_ : index; -} - -inline CordRepRing::index_type CordRepRing::retreat(index_type index) const { - assert(index < capacity_); - return (index > 0 ? index : capacity_) - 1; -} - -inline CordRepRing::index_type CordRepRing::retreat(index_type index, - index_type n) const { - assert(index < capacity_ && n <= capacity_); - return index >= n ? index - n : capacity_ - n + index; -} - -inline absl::string_view CordRepRing::entry_data(index_type index) const { - size_t data_offset = entry_data_offset(index); - return {GetRepData(entry_child(index)) + data_offset, entry_length(index)}; -} - -inline bool CordRepRing::IsValidIndex(index_type index) const { - if (index >= capacity_) return false; - return (tail_ > head_) ? (index >= head_ && index < tail_) - : (index >= head_ || index < tail_); -} - -#ifndef EXTRA_CORD_RING_VALIDATION -inline CordRepRing* CordRepRing::Validate(CordRepRing* rep, - const char* /*file*/, int /*line*/) { - return rep; -} -#endif - -inline CordRepRing::Position CordRepRing::Find(size_t offset) const { - assert(offset < length); - return (offset == 0) ? Position{head_, 0} : FindSlow(head_, offset); -} - -inline CordRepRing::Position CordRepRing::Find(index_type head, - size_t offset) const { - assert(offset < length); - assert(IsValidIndex(head) && offset >= entry_start_offset(head)); - return (offset == 0) ? Position{head_, 0} : FindSlow(head, offset); -} - -inline CordRepRing::Position CordRepRing::FindTail(size_t offset) const { - assert(offset > 0 && offset <= length); - return (offset == length) ? Position{tail_, 0} : FindTailSlow(head_, offset); -} - -inline CordRepRing::Position CordRepRing::FindTail(index_type head, - size_t offset) const { - assert(offset > 0 && offset <= length); - assert(IsValidIndex(head) && offset >= entry_start_offset(head) + 1); - return (offset == length) ? Position{tail_, 0} : FindTailSlow(head, offset); -} - -// Now that CordRepRing is defined, we can define CordRep's helper casts: -inline CordRepRing* CordRep::ring() { - assert(IsRing()); - return static_cast<CordRepRing*>(this); -} - -inline const CordRepRing* CordRep::ring() const { - assert(IsRing()); - return static_cast<const CordRepRing*>(this); -} - -inline bool CordRepRing::IsFlat(absl::string_view* fragment) const { - if (entries() == 1) { - if (fragment) *fragment = entry_data(head()); - return true; - } - return false; -} - -inline bool CordRepRing::IsFlat(size_t offset, size_t len, - absl::string_view* fragment) const { - const Position pos = Find(offset); - const absl::string_view data = entry_data(pos.index); - if (data.length() >= len && data.length() - len >= pos.offset) { - if (fragment) *fragment = data.substr(pos.offset, len); - return true; - } - return false; -} - -std::ostream& operator<<(std::ostream& s, const CordRepRing& rep); - -} // namespace cord_internal -ABSL_NAMESPACE_END -} // namespace absl - -#endif // ABSL_STRINGS_INTERNAL_CORD_REP_RING_H_ diff --git a/contrib/restricted/abseil-cpp/absl/strings/internal/cordz_info.cc b/contrib/restricted/abseil-cpp/absl/strings/internal/cordz_info.cc index 515dfafbb3..b24c3da712 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/internal/cordz_info.cc +++ b/contrib/restricted/abseil-cpp/absl/strings/internal/cordz_info.cc @@ -21,7 +21,6 @@ #include "absl/strings/internal/cord_internal.h" #include "absl/strings/internal/cord_rep_btree.h" #include "absl/strings/internal/cord_rep_crc.h" -#include "absl/strings/internal/cord_rep_ring.h" #include "absl/strings/internal/cordz_handle.h" #include "absl/strings/internal/cordz_statistics.h" #include "absl/strings/internal/cordz_update_tracker.h" @@ -33,8 +32,6 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace cord_internal { -using ::absl::base_internal::SpinLockHolder; - #ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL constexpr size_t CordzInfo::kMaxStackDepth; #endif @@ -79,6 +76,8 @@ class CordRepAnalyzer { // adds the results to `statistics`. Note that node counts and memory sizes // are not initialized, computed values are added to any existing values. void AnalyzeCordRep(const CordRep* rep) { + ABSL_ASSERT(rep != nullptr); + // Process all linear nodes. // As per the class comments, use refcout - 1 on the top level node, as the // top level node is assumed to be referenced only for analysis purposes. @@ -86,7 +85,7 @@ class CordRepAnalyzer { RepRef repref{rep, (refcount > 1) ? refcount - 1 : 1}; // Process the top level CRC node, if present. - if (repref.rep->tag == CRC) { + if (repref.tag() == CRC) { statistics_.node_count++; statistics_.node_counts.crc++; memory_usage_.Add(sizeof(CordRepCrc), repref.refcount); @@ -96,15 +95,14 @@ class CordRepAnalyzer { // Process all top level linear nodes (substrings and flats). repref = CountLinearReps(repref, memory_usage_); - if (repref.rep != nullptr) { - if (repref.rep->tag == RING) { - AnalyzeRing(repref); - } else if (repref.rep->tag == BTREE) { + switch (repref.tag()) { + case CordRepKind::BTREE: AnalyzeBtree(repref); - } else { - // We should have either a concat, btree, or ring node if not null. - assert(false); - } + break; + default: + // We should have a btree node if not null. + ABSL_ASSERT(repref.tag() == CordRepKind::UNUSED_0); + break; } // Adds values to output @@ -122,11 +120,19 @@ class CordRepAnalyzer { const CordRep* rep; size_t refcount; - // Returns a 'child' RepRef which contains the cumulative reference count of - // this instance multiplied by the child's reference count. + // Returns a 'child' RepRef which contains the cumulative reference count + // of this instance multiplied by the child's reference count. Returns a + // nullptr RepRef value with a refcount of 0 if `child` is nullptr. RepRef Child(const CordRep* child) const { + if (child == nullptr) return RepRef{nullptr, 0}; return RepRef{child, refcount * child->refcount.Get()}; } + + // Returns the tag of this rep, or UNUSED_0 if this instance is null + constexpr CordRepKind tag() const { + ABSL_ASSERT(rep == nullptr || rep->tag != CordRepKind::UNUSED_0); + return rep ? static_cast<CordRepKind>(rep->tag) : CordRepKind::UNUSED_0; + } }; // Memory usage values @@ -167,7 +173,7 @@ class CordRepAnalyzer { // buffers where we count children unrounded. RepRef CountLinearReps(RepRef rep, MemoryUsage& memory_usage) { // Consume all substrings - while (rep.rep->tag == SUBSTRING) { + while (rep.tag() == SUBSTRING) { statistics_.node_count++; statistics_.node_counts.substring++; memory_usage.Add(sizeof(CordRepSubstring), rep.refcount); @@ -175,7 +181,7 @@ class CordRepAnalyzer { } // Consume possible FLAT - if (rep.rep->tag >= FLAT) { + if (rep.tag() >= FLAT) { size_t size = rep.rep->flat()->AllocatedSize(); CountFlat(size); memory_usage.Add(size, rep.refcount); @@ -183,7 +189,7 @@ class CordRepAnalyzer { } // Consume possible external - if (rep.rep->tag == EXTERNAL) { + if (rep.tag() == EXTERNAL) { statistics_.node_count++; statistics_.node_counts.external++; size_t size = rep.rep->length + sizeof(CordRepExternalImpl<intptr_t>); @@ -194,17 +200,6 @@ class CordRepAnalyzer { return rep; } - // Analyzes the provided ring. - void AnalyzeRing(RepRef rep) { - statistics_.node_count++; - statistics_.node_counts.ring++; - const CordRepRing* ring = rep.rep->ring(); - memory_usage_.Add(CordRepRing::AllocSize(ring->capacity()), rep.refcount); - ring->ForEach([&](CordRepRing::index_type pos) { - CountLinearReps(rep.Child(ring->entry_child(pos)), memory_usage_); - }); - } - // Analyzes the provided btree. void AnalyzeBtree(RepRef rep) { statistics_.node_count++; diff --git a/contrib/restricted/abseil-cpp/absl/strings/internal/memutil.cc b/contrib/restricted/abseil-cpp/absl/strings/internal/memutil.cc index e2e7347c49..0bbd8aa1bd 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/internal/memutil.cc +++ b/contrib/restricted/abseil-cpp/absl/strings/internal/memutil.cc @@ -27,10 +27,18 @@ int memcasecmp(const char* s1, const char* s2, size_t len) { const unsigned char* us2 = reinterpret_cast<const unsigned char*>(s2); for (size_t i = 0; i < len; i++) { - const int diff = - int{static_cast<unsigned char>(absl::ascii_tolower(us1[i]))} - - int{static_cast<unsigned char>(absl::ascii_tolower(us2[i]))}; - if (diff != 0) return diff; + unsigned char c1 = us1[i]; + unsigned char c2 = us2[i]; + // If bytes are the same, they will be the same when converted to lower. + // So we only need to convert if bytes are not equal. + // NOTE(b/308193381): We do not use `absl::ascii_tolower` here in order + // to avoid its lookup table and improve performance. + if (c1 != c2) { + c1 = c1 >= 'A' && c1 <= 'Z' ? c1 - 'A' + 'a' : c1; + c2 = c2 >= 'A' && c2 <= 'Z' ? c2 - 'A' + 'a' : c2; + const int diff = int{c1} - int{c2}; + if (diff != 0) return diff; + } } return 0; } diff --git a/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/arg.cc b/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/arg.cc index c0a9a28ef9..eeb2108154 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/arg.cc +++ b/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/arg.cc @@ -18,15 +18,28 @@ // #include "absl/strings/internal/str_format/arg.h" +#include <algorithm> #include <cassert> -#include <cerrno> +#include <cstddef> +#include <cstdint> #include <cstdlib> +#include <cstring> +#include <cwchar> #include <string> #include <type_traits> -#include "absl/base/port.h" +#include "absl/base/config.h" +#include "absl/base/optimization.h" +#include "absl/container/fixed_array.h" +#include "absl/numeric/int128.h" +#include "absl/strings/internal/str_format/extension.h" #include "absl/strings/internal/str_format/float_conversion.h" #include "absl/strings/numbers.h" +#include "absl/strings/string_view.h" + +#if defined(ABSL_HAVE_STD_STRING_VIEW) +#include <string_view> +#endif namespace absl { ABSL_NAMESPACE_BEGIN @@ -298,6 +311,83 @@ inline bool ConvertStringArg(string_view v, const FormatConversionSpecImpl conv, conv.has_left_flag()); } +struct ShiftState { + bool saw_high_surrogate = false; + uint8_t bits = 0; +}; + +// Converts `v` from UTF-16 or UTF-32 to UTF-8 and writes to `buf`. `buf` is +// assumed to have enough space for the output. `s` is used to carry state +// between successive calls with a UTF-16 surrogate pair. Returns the number of +// chars written, or `static_cast<size_t>(-1)` on failure. +// +// This is basically std::wcrtomb(), but always outputting UTF-8 instead of +// respecting the current locale. +inline size_t WideToUtf8(wchar_t wc, char *buf, ShiftState &s) { + const auto v = static_cast<uint32_t>(wc); + if (v < 0x80) { + *buf = static_cast<char>(v); + return 1; + } else if (v < 0x800) { + *buf++ = static_cast<char>(0xc0 | (v >> 6)); + *buf = static_cast<char>(0x80 | (v & 0x3f)); + return 2; + } else if (v < 0xd800 || (v - 0xe000) < 0x2000) { + *buf++ = static_cast<char>(0xe0 | (v >> 12)); + *buf++ = static_cast<char>(0x80 | ((v >> 6) & 0x3f)); + *buf = static_cast<char>(0x80 | (v & 0x3f)); + return 3; + } else if ((v - 0x10000) < 0x100000) { + *buf++ = static_cast<char>(0xf0 | (v >> 18)); + *buf++ = static_cast<char>(0x80 | ((v >> 12) & 0x3f)); + *buf++ = static_cast<char>(0x80 | ((v >> 6) & 0x3f)); + *buf = static_cast<char>(0x80 | (v & 0x3f)); + return 4; + } else if (v < 0xdc00) { + s.saw_high_surrogate = true; + s.bits = static_cast<uint8_t>(v & 0x3); + const uint8_t high_bits = ((v >> 6) & 0xf) + 1; + *buf++ = static_cast<char>(0xf0 | (high_bits >> 2)); + *buf = + static_cast<char>(0x80 | static_cast<uint8_t>((high_bits & 0x3) << 4) | + static_cast<uint8_t>((v >> 2) & 0xf)); + return 2; + } else if (v < 0xe000 && s.saw_high_surrogate) { + *buf++ = static_cast<char>(0x80 | static_cast<uint8_t>(s.bits << 4) | + static_cast<uint8_t>((v >> 6) & 0xf)); + *buf = static_cast<char>(0x80 | (v & 0x3f)); + s.saw_high_surrogate = false; + s.bits = 0; + return 2; + } else { + return static_cast<size_t>(-1); + } +} + +inline bool ConvertStringArg(const wchar_t *v, + size_t len, + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + FixedArray<char> mb(len * 4); + ShiftState s; + size_t chars_written = 0; + for (size_t i = 0; i < len; ++i) { + const size_t chars = WideToUtf8(v[i], &mb[chars_written], s); + if (chars == static_cast<size_t>(-1)) { return false; } + chars_written += chars; + } + return ConvertStringArg(string_view(mb.data(), chars_written), conv, sink); +} + +bool ConvertWCharTImpl(wchar_t v, const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + char mb[4]; + ShiftState s; + const size_t chars_written = WideToUtf8(v, mb, s); + return chars_written != static_cast<size_t>(-1) && !s.saw_high_surrogate && + ConvertStringArg(string_view(mb, chars_written), conv, sink); +} + } // namespace bool ConvertBoolArg(bool v, FormatSinkImpl *sink) { @@ -316,11 +406,14 @@ bool ConvertIntArg(T v, FormatConversionSpecImpl conv, FormatSinkImpl *sink) { // This odd casting is due to a bug in -Wswitch behavior in gcc49 which causes // it to complain about a switch/case type mismatch, even though both are - // FormatConverionChar. Likely this is because at this point + // FormatConversionChar. Likely this is because at this point // FormatConversionChar is declared, but not defined. switch (static_cast<uint8_t>(conv.conversion_char())) { case static_cast<uint8_t>(FormatConversionCharInternal::c): - return ConvertCharImpl(static_cast<char>(v), conv, sink); + return (std::is_same<T, wchar_t>::value || + (conv.length_mod() == LengthMod::l)) + ? ConvertWCharTImpl(static_cast<wchar_t>(v), conv, sink) + : ConvertCharImpl(static_cast<char>(v), conv, sink); case static_cast<uint8_t>(FormatConversionCharInternal::o): as_digits.PrintAsOct(static_cast<U>(v)); @@ -372,6 +465,8 @@ template bool ConvertIntArg<signed char>(signed char v, template bool ConvertIntArg<unsigned char>(unsigned char v, FormatConversionSpecImpl conv, FormatSinkImpl *sink); +template bool ConvertIntArg<wchar_t>(wchar_t v, FormatConversionSpecImpl conv, + FormatSinkImpl *sink); template bool ConvertIntArg<short>(short v, // NOLINT FormatConversionSpecImpl conv, FormatSinkImpl *sink); @@ -403,16 +498,29 @@ StringConvertResult FormatConvertImpl(const std::string &v, return {ConvertStringArg(v, conv, sink)}; } +StringConvertResult FormatConvertImpl(const std::wstring &v, + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return {ConvertStringArg(v.data(), v.size(), conv, sink)}; +} + StringConvertResult FormatConvertImpl(string_view v, const FormatConversionSpecImpl conv, FormatSinkImpl *sink) { return {ConvertStringArg(v, conv, sink)}; } -ArgConvertResult<FormatConversionCharSetUnion( - FormatConversionCharSetInternal::s, FormatConversionCharSetInternal::p)> -FormatConvertImpl(const char *v, const FormatConversionSpecImpl conv, - FormatSinkImpl *sink) { +#if defined(ABSL_HAVE_STD_STRING_VIEW) +StringConvertResult FormatConvertImpl(std::wstring_view v, + const FormatConversionSpecImpl conv, + FormatSinkImpl* sink) { + return {ConvertStringArg(v.data(), v.size(), conv, sink)}; +} +#endif + +StringPtrConvertResult FormatConvertImpl(const char* v, + const FormatConversionSpecImpl conv, + FormatSinkImpl* sink) { if (conv.conversion_char() == FormatConversionCharInternal::p) return {FormatConvertImpl(VoidPtr(v), conv, sink).value}; size_t len; @@ -427,6 +535,30 @@ FormatConvertImpl(const char *v, const FormatConversionSpecImpl conv, return {ConvertStringArg(string_view(v, len), conv, sink)}; } +StringPtrConvertResult FormatConvertImpl(const wchar_t* v, + const FormatConversionSpecImpl conv, + FormatSinkImpl* sink) { + if (conv.conversion_char() == FormatConversionCharInternal::p) { + return {FormatConvertImpl(VoidPtr(v), conv, sink).value}; + } + size_t len; + if (v == nullptr) { + len = 0; + } else if (conv.precision() < 0) { + len = std::wcslen(v); + } else { + // If precision is set, we look for the NUL-terminator on the valid range. + len = static_cast<size_t>(std::find(v, v + conv.precision(), L'\0') - v); + } + return {ConvertStringArg(v, len, conv, sink)}; +} + +StringPtrConvertResult FormatConvertImpl(std::nullptr_t, + const FormatConversionSpecImpl conv, + FormatSinkImpl* sink) { + return FormatConvertImpl(static_cast<const char*>(nullptr), conv, sink); +} + // ==================== Raw pointers ==================== ArgConvertResult<FormatConversionCharSetInternal::p> FormatConvertImpl( VoidPtr v, const FormatConversionSpecImpl conv, FormatSinkImpl *sink) { @@ -461,6 +593,11 @@ CharConvertResult FormatConvertImpl(char v, const FormatConversionSpecImpl conv, FormatSinkImpl *sink) { return {ConvertIntArg(v, conv, sink)}; } +CharConvertResult FormatConvertImpl(wchar_t v, + const FormatConversionSpecImpl conv, + FormatSinkImpl* sink) { + return {ConvertIntArg(v, conv, sink)}; +} // ==================== Ints ==================== IntegralConvertResult FormatConvertImpl(signed char v, diff --git a/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/arg.h b/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/arg.h index 3ce30feb49..309161d591 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/arg.h +++ b/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/arg.h @@ -19,8 +19,9 @@ #include <wchar.h> #include <algorithm> +#include <cstddef> +#include <cstdint> #include <cstdio> -#include <iomanip> #include <limits> #include <memory> #include <sstream> @@ -28,13 +29,18 @@ #include <type_traits> #include <utility> -#include "absl/base/port.h" +#include "absl/base/config.h" +#include "absl/base/optimization.h" #include "absl/meta/type_traits.h" #include "absl/numeric/int128.h" -#include "absl/strings/internal/has_absl_stringify.h" +#include "absl/strings/has_absl_stringify.h" #include "absl/strings/internal/str_format/extension.h" #include "absl/strings/string_view.h" +#if defined(ABSL_HAVE_STD_STRING_VIEW) +#include <string_view> +#endif + namespace absl { ABSL_NAMESPACE_BEGIN @@ -97,6 +103,9 @@ extern template bool ConvertIntArg<signed char>(signed char v, extern template bool ConvertIntArg<unsigned char>(unsigned char v, FormatConversionSpecImpl conv, FormatSinkImpl* sink); +extern template bool ConvertIntArg<wchar_t>(wchar_t v, + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); extern template bool ConvertIntArg<short>(short v, // NOLINT FormatConversionSpecImpl conv, FormatSinkImpl* sink); @@ -158,6 +167,7 @@ template <typename T> auto FormatConvertImpl(const T& v, FormatConversionSpecImpl, FormatSinkImpl* sink) -> std::enable_if_t<!std::is_enum<T>::value && + !std::is_same<T, absl::Cord>::value && std::is_void<decltype(AbslStringify( std::declval<FormatSink&>(), v))>::value, ArgConvertResult<FormatConversionCharSetInternal::v>> { @@ -202,30 +212,49 @@ constexpr FormatConversionCharSet ExtractCharSet(ArgConvertResult<C>) { return C; } -using StringConvertResult = ArgConvertResult<FormatConversionCharSetUnion( - FormatConversionCharSetInternal::s, FormatConversionCharSetInternal::v)>; ArgConvertResult<FormatConversionCharSetInternal::p> FormatConvertImpl( VoidPtr v, FormatConversionSpecImpl conv, FormatSinkImpl* sink); // Strings. +using StringConvertResult = ArgConvertResult<FormatConversionCharSetUnion( + FormatConversionCharSetInternal::s, + FormatConversionCharSetInternal::v)>; StringConvertResult FormatConvertImpl(const std::string& v, FormatConversionSpecImpl conv, FormatSinkImpl* sink); +StringConvertResult FormatConvertImpl(const std::wstring& v, + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); StringConvertResult FormatConvertImpl(string_view v, FormatConversionSpecImpl conv, FormatSinkImpl* sink); -#if defined(ABSL_HAVE_STD_STRING_VIEW) && !defined(ABSL_USES_STD_STRING_VIEW) +#if defined(ABSL_HAVE_STD_STRING_VIEW) +StringConvertResult FormatConvertImpl(std::wstring_view v, + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); +#if !defined(ABSL_USES_STD_STRING_VIEW) inline StringConvertResult FormatConvertImpl(std::string_view v, FormatConversionSpecImpl conv, FormatSinkImpl* sink) { return FormatConvertImpl(absl::string_view(v.data(), v.size()), conv, sink); } -#endif // ABSL_HAVE_STD_STRING_VIEW && !ABSL_USES_STD_STRING_VIEW - -ArgConvertResult<FormatConversionCharSetUnion( - FormatConversionCharSetInternal::s, FormatConversionCharSetInternal::p)> -FormatConvertImpl(const char* v, const FormatConversionSpecImpl conv, - FormatSinkImpl* sink); +#endif // !ABSL_USES_STD_STRING_VIEW +#endif // ABSL_HAVE_STD_STRING_VIEW + +using StringPtrConvertResult = ArgConvertResult<FormatConversionCharSetUnion( + FormatConversionCharSetInternal::s, + FormatConversionCharSetInternal::p)>; +StringPtrConvertResult FormatConvertImpl(const char* v, + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); +StringPtrConvertResult FormatConvertImpl(const wchar_t* v, + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); +// This overload is needed to disambiguate, since `nullptr` could match either +// of the other overloads equally well. +StringPtrConvertResult FormatConvertImpl(std::nullptr_t, + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); template <class AbslCord, typename std::enable_if<std::is_same< AbslCord, absl::Cord>::value>::type* = nullptr> @@ -279,6 +308,9 @@ FloatingConvertResult FormatConvertImpl(long double v, // Chars. CharConvertResult FormatConvertImpl(char v, FormatConversionSpecImpl conv, FormatSinkImpl* sink); +CharConvertResult FormatConvertImpl(wchar_t v, + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); // Ints. IntegralConvertResult FormatConvertImpl(signed char v, @@ -333,7 +365,7 @@ IntegralConvertResult FormatConvertImpl(T v, FormatConversionSpecImpl conv, template <typename T> typename std::enable_if<std::is_enum<T>::value && !HasUserDefinedConvert<T>::value && - !strings_internal::HasAbslStringify<T>::value, + !HasAbslStringify<T>::value, IntegralConvertResult>::type FormatConvertImpl(T v, FormatConversionSpecImpl conv, FormatSinkImpl* sink); @@ -440,6 +472,7 @@ class FormatArgImpl { // Anything with a user-defined Convert will get its own vtable. // For everything else: // - Decay char* and char arrays into `const char*` + // - Decay wchar_t* and wchar_t arrays into `const wchar_t*` // - Decay any other pointer to `const void*` // - Decay all enums to the integral promotion of their underlying type. // - Decay function pointers to void*. @@ -447,20 +480,23 @@ class FormatArgImpl { struct DecayType { static constexpr bool kHasUserDefined = str_format_internal::HasUserDefinedConvert<T>::value || - strings_internal::HasAbslStringify<T>::value; + HasAbslStringify<T>::value; using type = typename std::conditional< !kHasUserDefined && std::is_convertible<T, const char*>::value, const char*, - typename std::conditional<!kHasUserDefined && - std::is_convertible<T, VoidPtr>::value, - VoidPtr, const T&>::type>::type; + typename std::conditional< + !kHasUserDefined && std::is_convertible<T, const wchar_t*>::value, + const wchar_t*, + typename std::conditional< + !kHasUserDefined && std::is_convertible<T, VoidPtr>::value, + VoidPtr, + const T&>::type>::type>::type; }; template <typename T> - struct DecayType<T, - typename std::enable_if< - !str_format_internal::HasUserDefinedConvert<T>::value && - !strings_internal::HasAbslStringify<T>::value && - std::is_enum<T>::value>::type> { + struct DecayType< + T, typename std::enable_if< + !str_format_internal::HasUserDefinedConvert<T>::value && + !HasAbslStringify<T>::value && std::is_enum<T>::value>::type> { using type = decltype(+typename std::underlying_type<T>::type()); }; @@ -585,7 +621,7 @@ class FormatArgImpl { E template bool FormatArgImpl::Dispatch<T>(Data, FormatConversionSpecImpl, \ void*) -#define ABSL_INTERNAL_FORMAT_DISPATCH_OVERLOADS_EXPAND_(...) \ +#define ABSL_INTERNAL_FORMAT_DISPATCH_OVERLOADS_EXPAND_NO_WSTRING_VIEW_(...) \ ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(str_format_internal::VoidPtr, \ __VA_ARGS__); \ ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(bool, __VA_ARGS__); \ @@ -611,7 +647,19 @@ class FormatArgImpl { ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(long double, __VA_ARGS__); \ ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(const char*, __VA_ARGS__); \ ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(std::string, __VA_ARGS__); \ - ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(string_view, __VA_ARGS__) + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(string_view, __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(const wchar_t*, __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(std::wstring, __VA_ARGS__) + +#if defined(ABSL_HAVE_STD_STRING_VIEW) +#define ABSL_INTERNAL_FORMAT_DISPATCH_OVERLOADS_EXPAND_(...) \ + ABSL_INTERNAL_FORMAT_DISPATCH_OVERLOADS_EXPAND_NO_WSTRING_VIEW_( \ + __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(std::wstring_view, __VA_ARGS__) +#else +#define ABSL_INTERNAL_FORMAT_DISPATCH_OVERLOADS_EXPAND_(...) \ + ABSL_INTERNAL_FORMAT_DISPATCH_OVERLOADS_EXPAND_NO_WSTRING_VIEW_(__VA_ARGS__) +#endif ABSL_INTERNAL_FORMAT_DISPATCH_OVERLOADS_EXPAND_(extern); diff --git a/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/bind.cc b/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/bind.cc index 77a4222337..87e23b565c 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/bind.cc +++ b/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/bind.cc @@ -14,10 +14,24 @@ #include "absl/strings/internal/str_format/bind.h" +#include <algorithm> +#include <cassert> #include <cerrno> +#include <cstddef> +#include <cstdio> +#include <ios> #include <limits> +#include <ostream> #include <sstream> #include <string> +#include "absl/base/config.h" +#include "absl/base/optimization.h" +#include "absl/strings/internal/str_format/arg.h" +#include "absl/strings/internal/str_format/constexpr_parser.h" +#include "absl/strings/internal/str_format/extension.h" +#include "absl/strings/internal/str_format/output.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -90,6 +104,8 @@ inline bool ArgContext::Bind(const UnboundConversion* unbound, } else { FormatConversionSpecImplFriend::SetFlags(unbound->flags, bound); } + + FormatConversionSpecImplFriend::SetLengthMod(unbound->length_mod, bound); } else { FormatConversionSpecImplFriend::SetFlags(unbound->flags, bound); FormatConversionSpecImplFriend::SetWidth(-1, bound); @@ -215,7 +231,7 @@ std::string& AppendPack(std::string* out, const UntypedFormatSpecImpl format, return *out; } -std::string FormatPack(const UntypedFormatSpecImpl format, +std::string FormatPack(UntypedFormatSpecImpl format, absl::Span<const FormatArgImpl> args) { std::string out; if (ABSL_PREDICT_FALSE(!FormatUntyped(&out, format, args))) { diff --git a/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/bind.h b/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/bind.h index 5e2a43d5c9..120bc355c7 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/bind.h +++ b/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/bind.h @@ -15,16 +15,19 @@ #ifndef ABSL_STRINGS_INTERNAL_STR_FORMAT_BIND_H_ #define ABSL_STRINGS_INTERNAL_STR_FORMAT_BIND_H_ -#include <array> +#include <cassert> #include <cstdio> -#include <sstream> +#include <ostream> #include <string> -#include "absl/base/port.h" +#include "absl/base/config.h" #include "absl/container/inlined_vector.h" #include "absl/strings/internal/str_format/arg.h" #include "absl/strings/internal/str_format/checker.h" +#include "absl/strings/internal/str_format/constexpr_parser.h" +#include "absl/strings/internal/str_format/extension.h" #include "absl/strings/internal/str_format/parser.h" +#include "absl/strings/string_view.h" #include "absl/types/span.h" #include "absl/utility/utility.h" @@ -203,7 +206,7 @@ bool FormatUntyped(FormatRawSinkImpl raw_sink, UntypedFormatSpecImpl format, std::string& AppendPack(std::string* out, UntypedFormatSpecImpl format, absl::Span<const FormatArgImpl> args); -std::string FormatPack(const UntypedFormatSpecImpl format, +std::string FormatPack(UntypedFormatSpecImpl format, absl::Span<const FormatArgImpl> args); int FprintF(std::FILE* output, UntypedFormatSpecImpl format, diff --git a/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/constexpr_parser.h b/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/constexpr_parser.h index b70a16e40c..8f59387047 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/constexpr_parser.h +++ b/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/constexpr_parser.h @@ -17,17 +17,18 @@ #include <cassert> #include <cstdint> +#include <cstdio> #include <limits> +#include "absl/base/config.h" #include "absl/base/const_init.h" +#include "absl/base/optimization.h" #include "absl/strings/internal/str_format/extension.h" namespace absl { ABSL_NAMESPACE_BEGIN namespace str_format_internal { -enum class LengthMod : std::uint8_t { h, hh, l, ll, L, j, z, t, q, none }; - // The analyzed properties of a single specified conversion. struct UnboundConversion { // This is a user defined default constructor on purpose to skip the @@ -306,7 +307,6 @@ constexpr const char* ConsumeConversion(const char* pos, const char* const end, if (ABSL_PREDICT_FALSE(!tag.is_length())) return nullptr; // It is a length modifier. - using str_format_internal::LengthMod; LengthMod length_mod = tag.as_length(); ABSL_FORMAT_PARSER_INTERNAL_GET_CHAR(); if (c == 'h' && length_mod == LengthMod::h) { @@ -322,6 +322,11 @@ constexpr const char* ConsumeConversion(const char* pos, const char* const end, if (ABSL_PREDICT_FALSE(c == 'v')) return nullptr; if (ABSL_PREDICT_FALSE(!tag.is_conv())) return nullptr; + + // `wchar_t` args are marked non-basic so `Bind()` will copy the length mod. + if (conv->length_mod == LengthMod::l && c == 'c') { + conv->flags = conv->flags | Flags::kNonBasic; + } } #undef ABSL_FORMAT_PARSER_INTERNAL_GET_CHAR diff --git a/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/extension.h b/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/extension.h index 8de42d2cfc..173284c6c7 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/extension.h +++ b/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/extension.h @@ -16,16 +16,14 @@ #ifndef ABSL_STRINGS_INTERNAL_STR_FORMAT_EXTENSION_H_ #define ABSL_STRINGS_INTERNAL_STR_FORMAT_EXTENSION_H_ -#include <limits.h> #include <cstddef> #include <cstdint> #include <cstring> #include <ostream> +#include <string> #include "absl/base/config.h" -#include "absl/base/port.h" -#include "absl/meta/type_traits.h" #include "absl/strings/internal/str_format/output.h" #include "absl/strings/string_view.h" @@ -34,6 +32,7 @@ ABSL_NAMESPACE_BEGIN enum class FormatConversionChar : uint8_t; enum class FormatConversionCharSet : uint64_t; +enum class LengthMod : std::uint8_t { h, hh, l, ll, L, j, z, t, q, none }; namespace str_format_internal { @@ -139,7 +138,8 @@ enum class Flags : uint8_t { kAlt = 1 << 3, kZero = 1 << 4, // This is not a real flag. It just exists to turn off kBasic when no other - // flags are set. This is for when width/precision are specified. + // flags are set. This is for when width/precision are specified, or a length + // modifier affects the behavior ("%lc"). kNonBasic = 1 << 5, }; @@ -285,6 +285,8 @@ class FormatConversionSpecImpl { bool has_alt_flag() const { return FlagsContains(flags_, Flags::kAlt); } bool has_zero_flag() const { return FlagsContains(flags_, Flags::kZero); } + LengthMod length_mod() const { return length_mod_; } + FormatConversionChar conversion_char() const { // Keep this field first in the struct . It generates better code when // accessing it when ConversionSpec is passed by value in registers. @@ -310,6 +312,7 @@ class FormatConversionSpecImpl { friend struct str_format_internal::FormatConversionSpecImplFriend; FormatConversionChar conv_ = FormatConversionCharInternal::kNone; Flags flags_; + LengthMod length_mod_ = LengthMod::none; int width_; int precision_; }; @@ -318,6 +321,9 @@ struct FormatConversionSpecImplFriend final { static void SetFlags(Flags f, FormatConversionSpecImpl* conv) { conv->flags_ = f; } + static void SetLengthMod(LengthMod l, FormatConversionSpecImpl* conv) { + conv->length_mod_ = l; + } static void SetConversionChar(FormatConversionChar c, FormatConversionSpecImpl* conv) { conv->conv_ = c; diff --git a/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/parser.h b/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/parser.h index 35b6d49c14..b1d6d5fd0e 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/parser.h +++ b/contrib/restricted/abseil-cpp/absl/strings/internal/str_format/parser.h @@ -15,22 +15,23 @@ #ifndef ABSL_STRINGS_INTERNAL_STR_FORMAT_PARSER_H_ #define ABSL_STRINGS_INTERNAL_STR_FORMAT_PARSER_H_ -#include <limits.h> #include <stddef.h> #include <stdlib.h> #include <cassert> -#include <cstdint> +#include <cstring> #include <initializer_list> -#include <iosfwd> -#include <iterator> #include <memory> #include <string> +#include <utility> #include <vector> +#include "absl/base/config.h" +#include "absl/base/optimization.h" #include "absl/strings/internal/str_format/checker.h" #include "absl/strings/internal/str_format/constexpr_parser.h" #include "absl/strings/internal/str_format/extension.h" +#include "absl/strings/string_view.h" namespace absl { ABSL_NAMESPACE_BEGIN diff --git a/contrib/restricted/abseil-cpp/absl/strings/match.cc b/contrib/restricted/abseil-cpp/absl/strings/match.cc index 3b81b2c052..72ae6a430a 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/match.cc +++ b/contrib/restricted/abseil-cpp/absl/strings/match.cc @@ -17,10 +17,13 @@ #include <algorithm> #include <cstdint> +#include "absl/base/config.h" #include "absl/base/internal/endian.h" +#include "absl/base/optimization.h" #include "absl/numeric/bits.h" #include "absl/strings/ascii.h" #include "absl/strings/internal/memutil.h" +#include "absl/strings/string_view.h" namespace absl { ABSL_NAMESPACE_BEGIN diff --git a/contrib/restricted/abseil-cpp/absl/strings/numbers.cc b/contrib/restricted/abseil-cpp/absl/strings/numbers.cc index c43c6bcc19..882c3a8b85 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/numbers.cc +++ b/contrib/restricted/abseil-cpp/absl/strings/numbers.cc @@ -20,32 +20,36 @@ #include <algorithm> #include <cassert> #include <cfloat> // for DBL_DIG and FLT_DIG +#include <climits> #include <cmath> // for HUGE_VAL +#include <cstddef> #include <cstdint> #include <cstdio> #include <cstdlib> #include <cstring> #include <iterator> #include <limits> -#include <memory> +#include <system_error> // NOLINT(build/c++11) +#include <type_traits> #include <utility> #include "absl/base/attributes.h" +#include "absl/base/config.h" #include "absl/base/internal/endian.h" #include "absl/base/internal/raw_logging.h" +#include "absl/base/nullability.h" #include "absl/base/optimization.h" #include "absl/numeric/bits.h" +#include "absl/numeric/int128.h" #include "absl/strings/ascii.h" #include "absl/strings/charconv.h" -#include "absl/strings/escaping.h" -#include "absl/strings/internal/memutil.h" #include "absl/strings/match.h" -#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" namespace absl { ABSL_NAMESPACE_BEGIN -bool SimpleAtof(absl::string_view str, float* out) { +bool SimpleAtof(absl::string_view str, absl::Nonnull<float*> out) { *out = 0.0; str = StripAsciiWhitespace(str); // std::from_chars doesn't accept an initial +, but SimpleAtof does, so if one @@ -76,7 +80,7 @@ bool SimpleAtof(absl::string_view str, float* out) { return true; } -bool SimpleAtod(absl::string_view str, double* out) { +bool SimpleAtod(absl::string_view str, absl::Nonnull<double*> out) { *out = 0.0; str = StripAsciiWhitespace(str); // std::from_chars doesn't accept an initial +, but SimpleAtod does, so if one @@ -107,7 +111,7 @@ bool SimpleAtod(absl::string_view str, double* out) { return true; } -bool SimpleAtob(absl::string_view str, bool* out) { +bool SimpleAtob(absl::string_view str, absl::Nonnull<bool*> out) { ABSL_RAW_CHECK(out != nullptr, "Output pointer must not be nullptr."); if (EqualsIgnoreCase(str, "true") || EqualsIgnoreCase(str, "t") || EqualsIgnoreCase(str, "yes") || EqualsIgnoreCase(str, "y") || @@ -155,29 +159,71 @@ constexpr uint32_t kTwoZeroBytes = 0x0101 * '0'; constexpr uint64_t kFourZeroBytes = 0x01010101 * '0'; constexpr uint64_t kEightZeroBytes = 0x0101010101010101ull * '0'; -// * 103 / 1024 is a division by 10 for values from 0 to 99. It's also a -// division of a structure [k takes 2 bytes][m takes 2 bytes], then * 103 / 1024 -// will be [k / 10][m / 10]. It allows parallel division. -constexpr uint64_t kDivisionBy10Mul = 103u; +template <typename T> +constexpr T Pow(T base, uint32_t n) { + // Exponentiation by squaring + return static_cast<T>((n > 1 ? Pow(base * base, n >> 1) : static_cast<T>(1)) * + ((n & 1) ? base : static_cast<T>(1))); +} + +// Given n, calculates C where the following holds for all 0 <= x < Pow(100, n): +// x / Pow(10, n) == x * C / Pow(2, n * 10) +// In other words, it allows us to divide by a power of 10 via a single +// multiplication and bit shifts, assuming the input will be smaller than the +// square of that power of 10. +template <typename T> +constexpr T ComputePowerOf100DivisionCoefficient(uint32_t n) { + if (n > 4) { + // This doesn't work for large powers of 100, due to overflow + abort(); + } + T denom = 16 - 1; + T num = (denom + 1) - 10; + T gcd = 3; // Greatest common divisor of numerator and denominator + denom = Pow(denom / gcd, n); + num = Pow(num / gcd, 9 * n); + T quotient = num / denom; + if (num % denom >= denom / 2) { + // Round up, since the remainder is more than half the denominator + ++quotient; + } + return quotient; +} + +// * kDivisionBy10Mul / kDivisionBy10Div is a division by 10 for values from 0 +// to 99. It's also a division of a structure [k takes 2 bytes][m takes 2 +// bytes], then * kDivisionBy10Mul / kDivisionBy10Div will be [k / 10][m / 10]. +// It allows parallel division. +constexpr uint64_t kDivisionBy10Mul = + ComputePowerOf100DivisionCoefficient<uint64_t>(1); +static_assert(kDivisionBy10Mul == 103, + "division coefficient for 10 is incorrect"); constexpr uint64_t kDivisionBy10Div = 1 << 10; -// * 10486 / 1048576 is a division by 100 for values from 0 to 9999. -constexpr uint64_t kDivisionBy100Mul = 10486u; +// * kDivisionBy100Mul / kDivisionBy100Div is a division by 100 for values from +// 0 to 9999. +constexpr uint64_t kDivisionBy100Mul = + ComputePowerOf100DivisionCoefficient<uint64_t>(2); +static_assert(kDivisionBy100Mul == 10486, + "division coefficient for 100 is incorrect"); constexpr uint64_t kDivisionBy100Div = 1 << 20; -// Encode functions write the ASCII output of input `n` to `out_str`. -inline char* EncodeHundred(uint32_t n, char* out_str) { - int num_digits = static_cast<int>(n - 10) >> 8; - uint32_t base = kTwoZeroBytes; - uint32_t div10 = (n * kDivisionBy10Mul) / kDivisionBy10Div; - uint32_t mod10 = n - 10u * div10; - base += div10 + (mod10 << 8); - base >>= num_digits & 8; - little_endian::Store16(out_str, static_cast<uint16_t>(base)); - return out_str + 2 + num_digits; +static_assert(ComputePowerOf100DivisionCoefficient<uint64_t>(3) == 1073742, + "division coefficient for 1000 is incorrect"); + +// Same as `PrepareEightDigits`, but produces 2 digits for integers < 100. +inline uint32_t PrepareTwoDigitsImpl(uint32_t i, bool reversed) { + assert(i < 100); + uint32_t div10 = (i * kDivisionBy10Mul) / kDivisionBy10Div; + uint32_t mod10 = i - 10u * div10; + return (div10 << (reversed ? 8 : 0)) + (mod10 << (reversed ? 0 : 8)); +} +inline uint32_t PrepareTwoDigits(uint32_t i) { + return PrepareTwoDigitsImpl(i, false); } -inline char* EncodeTenThousand(uint32_t n, char* out_str) { +// Same as `PrepareEightDigits`, but produces 4 digits for integers < 10000. +inline uint32_t PrepareFourDigitsImpl(uint32_t n, bool reversed) { // We split lower 2 digits and upper 2 digits of n into 2 byte consecutive // blocks. 123 -> [\0\1][\0\23]. We divide by 10 both blocks // (it's 1 division + zeroing upper bits), and compute modulo 10 as well "in @@ -185,146 +231,335 @@ inline char* EncodeTenThousand(uint32_t n, char* out_str) { // strip trailing zeros, add ASCII '0000' and return. uint32_t div100 = (n * kDivisionBy100Mul) / kDivisionBy100Div; uint32_t mod100 = n - 100ull * div100; - uint32_t hundreds = (mod100 << 16) + div100; + uint32_t hundreds = + (mod100 << (reversed ? 0 : 16)) + (div100 << (reversed ? 16 : 0)); uint32_t tens = (hundreds * kDivisionBy10Mul) / kDivisionBy10Div; tens &= (0xFull << 16) | 0xFull; - tens += (hundreds - 10ull * tens) << 8; - ABSL_ASSUME(tens != 0); - // The result can contain trailing zero bits, we need to strip them to a first - // significant byte in a final representation. For example, for n = 123, we - // have tens to have representation \0\1\2\3. We do `& -8` to round - // to a multiple to 8 to strip zero bytes, not all zero bits. - // countr_zero to help. - // 0 minus 8 to make MSVC happy. - uint32_t zeroes = static_cast<uint32_t>(absl::countr_zero(tens)) & (0 - 8ull); - tens += kFourZeroBytes; - tens >>= zeroes; - little_endian::Store32(out_str, tens); - return out_str + sizeof(tens) - zeroes / 8; + tens = (tens << (reversed ? 8 : 0)) + + static_cast<uint32_t>((hundreds - 10ull * tens) << (reversed ? 0 : 8)); + return tens; +} +inline uint32_t PrepareFourDigits(uint32_t n) { + return PrepareFourDigitsImpl(n, false); +} +inline uint32_t PrepareFourDigitsReversed(uint32_t n) { + return PrepareFourDigitsImpl(n, true); } -// Prepare functions return an integer that should be written to out_str -// (but possibly include trailing zeros). -// For hi < 10000, lo < 10000 returns uint64_t as encoded in ASCII with -// possibly trailing zeroes of the number hi * 10000 + lo. -inline uint64_t PrepareTenThousands(uint64_t hi, uint64_t lo) { - uint64_t merged = hi | (lo << 32); +// Helper function to produce an ASCII representation of `i`. +// +// Function returns an 8-byte integer which when summed with `kEightZeroBytes`, +// can be treated as a printable buffer with ascii representation of `i`, +// possibly with leading zeros. +// +// Example: +// +// uint64_t buffer = PrepareEightDigits(102030) + kEightZeroBytes; +// char* ascii = reinterpret_cast<char*>(&buffer); +// // Note two leading zeros: +// EXPECT_EQ(absl::string_view(ascii, 8), "00102030"); +// +// If `Reversed` is set to true, the result becomes reversed to "03020100". +// +// Pre-condition: `i` must be less than 100000000. +inline uint64_t PrepareEightDigitsImpl(uint32_t i, bool reversed) { + ABSL_ASSUME(i < 10000'0000); + // Prepare 2 blocks of 4 digits "in parallel". + uint32_t hi = i / 10000; + uint32_t lo = i % 10000; + uint64_t merged = (uint64_t{hi} << (reversed ? 32 : 0)) | + (uint64_t{lo} << (reversed ? 0 : 32)); uint64_t div100 = ((merged * kDivisionBy100Mul) / kDivisionBy100Div) & ((0x7Full << 32) | 0x7Full); uint64_t mod100 = merged - 100ull * div100; - uint64_t hundreds = (mod100 << 16) + div100; + uint64_t hundreds = + (mod100 << (reversed ? 0 : 16)) + (div100 << (reversed ? 16 : 0)); uint64_t tens = (hundreds * kDivisionBy10Mul) / kDivisionBy10Div; tens &= (0xFull << 48) | (0xFull << 32) | (0xFull << 16) | 0xFull; - tens += (hundreds - 10ull * tens) << 8; + tens = (tens << (reversed ? 8 : 0)) + + ((hundreds - 10ull * tens) << (reversed ? 0 : 8)); return tens; } +inline uint64_t PrepareEightDigits(uint32_t i) { + return PrepareEightDigitsImpl(i, false); +} +inline uint64_t PrepareEightDigitsReversed(uint32_t i) { + return PrepareEightDigitsImpl(i, true); +} + +template <typename T, typename BackwardIt> +class FastUIntToStringConverter { + static_assert( + std::is_same<T, decltype(+std::declval<T>())>::value, + "to avoid code bloat, only instantiate this for int and larger types"); + static_assert(std::is_unsigned<T>::value, + "this class is only for unsigned types"); + + public: + // Outputs the given number backward (like with std::copy_backward), + // starting from the end of the string. + // The number of digits in the number must have been already measured and + // passed *exactly*, otherwise the behavior is undefined. + // (This is an optimization, as calculating the number of digits again would + // slow down the hot path.) + // Returns an iterator to the start of the suffix that was appended. + static BackwardIt FastIntToBufferBackward(T v, BackwardIt end) { + // THIS IS A HOT FUNCTION with a very deliberate structure to exploit branch + // prediction and shorten the critical path for smaller numbers. + // Do not move around the if/else blocks or attempt to simplify it + // without benchmarking any changes. + + if (v < 10) { + goto AT_LEAST_1 /* NOTE: mandatory for the 0 case */; + } + if (v < 1000) { + goto AT_LEAST_10; + } + if (v < 10000000) { + goto AT_LEAST_1000; + } + + if (v >= 100000000 / 10) { + if (v >= 10000000000000000 / 10) { + DoFastIntToBufferBackward<8>(v, end); + } + DoFastIntToBufferBackward<8>(v, end); + } + + if (v >= 10000 / 10) { + AT_LEAST_1000: + DoFastIntToBufferBackward<4>(v, end); + } + + if (v >= 100 / 10) { + AT_LEAST_10: + DoFastIntToBufferBackward<2>(v, end); + } -inline char* EncodeFullU32(uint32_t n, char* out_str) { - if (n < 100'000'000) { - uint64_t bottom = PrepareTenThousands(n / 10000, n % 10000); - ABSL_ASSUME(bottom != 0); - // 0 minus 8 to make MSVC happy. - uint32_t zeroes = static_cast<uint32_t>(absl::countr_zero(bottom)) - & (0 - 8ull); - uint64_t bottom_res = bottom + kEightZeroBytes; - bottom_res >>= zeroes; - little_endian::Store64(out_str, bottom_res); - return out_str + sizeof(bottom) - zeroes / 8; + if (v >= 10 / 10) { + AT_LEAST_1: + end = DoFastIntToBufferBackward(v, end, std::integral_constant<int, 1>()); + } + return end; } - uint32_t top = n / 100'000'000; - n %= 100'000'000; - uint64_t bottom = PrepareTenThousands(n / 10000, n % 10000); - uint64_t bottom_res = bottom + kEightZeroBytes; - out_str = EncodeHundred(top, out_str); - little_endian::Store64(out_str, bottom_res); - return out_str + sizeof(bottom); -} -} // namespace + private: + // Only assume pointers are contiguous for now. String and vector iterators + // could be special-cased as well, but there's no need for them here. + // With C++20 we can probably switch to std::contiguous_iterator_tag. + static constexpr bool kIsContiguousIterator = + std::is_pointer<BackwardIt>::value; + + template <int Exponent> + static void DoFastIntToBufferBackward(T& v, BackwardIt& end) { + constexpr T kModulus = Pow<T>(10, Exponent); + T remainder = static_cast<T>(v % kModulus); + v = static_cast<T>(v / kModulus); + end = DoFastIntToBufferBackward(remainder, end, + std::integral_constant<int, Exponent>()); + } -void numbers_internal::PutTwoDigits(uint32_t i, char* buf) { - assert(i < 100); - uint32_t base = kTwoZeroBytes; - uint32_t div10 = (i * kDivisionBy10Mul) / kDivisionBy10Div; - uint32_t mod10 = i - 10u * div10; - base += div10 + (mod10 << 8); - little_endian::Store16(buf, static_cast<uint16_t>(base)); -} + static BackwardIt DoFastIntToBufferBackward(const T&, BackwardIt end, + std::integral_constant<int, 0>) { + return end; + } -char* numbers_internal::FastIntToBuffer(uint32_t n, char* out_str) { - if (n < 100) { - out_str = EncodeHundred(n, out_str); - goto set_last_zero; + static BackwardIt DoFastIntToBufferBackward(T v, BackwardIt end, + std::integral_constant<int, 1>) { + *--end = static_cast<char>('0' + v); + return DoFastIntToBufferBackward(v, end, std::integral_constant<int, 0>()); } - if (n < 10000) { - out_str = EncodeTenThousand(n, out_str); - goto set_last_zero; + + static BackwardIt DoFastIntToBufferBackward(T v, BackwardIt end, + std::integral_constant<int, 4>) { + if (kIsContiguousIterator) { + const uint32_t digits = + PrepareFourDigits(static_cast<uint32_t>(v)) + kFourZeroBytes; + end -= sizeof(digits); + little_endian::Store32(&*end, digits); + } else { + uint32_t digits = + PrepareFourDigitsReversed(static_cast<uint32_t>(v)) + kFourZeroBytes; + for (size_t i = 0; i < sizeof(digits); ++i) { + *--end = static_cast<char>(digits); + digits >>= CHAR_BIT; + } + } + return end; } - out_str = EncodeFullU32(n, out_str); -set_last_zero: - *out_str = '\0'; - return out_str; -} -char* numbers_internal::FastIntToBuffer(int32_t i, char* buffer) { - uint32_t u = static_cast<uint32_t>(i); - if (i < 0) { - *buffer++ = '-'; - // We need to do the negation in modular (i.e., "unsigned") - // arithmetic; MSVC++ apparently warns for plain "-u", so - // we write the equivalent expression "0 - u" instead. - u = 0 - u; + static BackwardIt DoFastIntToBufferBackward(T v, BackwardIt end, + std::integral_constant<int, 8>) { + if (kIsContiguousIterator) { + const uint64_t digits = + PrepareEightDigits(static_cast<uint32_t>(v)) + kEightZeroBytes; + end -= sizeof(digits); + little_endian::Store64(&*end, digits); + } else { + uint64_t digits = PrepareEightDigitsReversed(static_cast<uint32_t>(v)) + + kEightZeroBytes; + for (size_t i = 0; i < sizeof(digits); ++i) { + *--end = static_cast<char>(digits); + digits >>= CHAR_BIT; + } + } + return end; } - return numbers_internal::FastIntToBuffer(u, buffer); -} -char* numbers_internal::FastIntToBuffer(uint64_t i, char* buffer) { - uint32_t u32 = static_cast<uint32_t>(i); - if (u32 == i) return numbers_internal::FastIntToBuffer(u32, buffer); - - // 10**9 < 2**32 <= i < 10**10, we can do 2+8 - uint64_t div08 = i / 100'000'000ull; - uint64_t mod08 = i % 100'000'000ull; - uint64_t mod_result = - PrepareTenThousands(mod08 / 10000, mod08 % 10000) + kEightZeroBytes; - if (i < 10'000'000'000ull) { - buffer = EncodeHundred(static_cast<uint32_t>(div08), buffer); - little_endian::Store64(buffer, mod_result); - buffer += 8; - goto set_last_zero; + template <int Digits> + static BackwardIt DoFastIntToBufferBackward( + T v, BackwardIt end, std::integral_constant<int, Digits>) { + constexpr int kLogModulus = Digits - Digits / 2; + constexpr T kModulus = Pow(static_cast<T>(10), kLogModulus); + bool is_safe_to_use_division_trick = Digits <= 8; + T quotient, remainder; + if (is_safe_to_use_division_trick) { + constexpr uint64_t kCoefficient = + ComputePowerOf100DivisionCoefficient<uint64_t>(kLogModulus); + quotient = (v * kCoefficient) >> (10 * kLogModulus); + remainder = v - quotient * kModulus; + } else { + quotient = v / kModulus; + remainder = v % kModulus; + } + end = DoFastIntToBufferBackward(remainder, end, + std::integral_constant<int, kLogModulus>()); + return DoFastIntToBufferBackward( + quotient, end, std::integral_constant<int, Digits - kLogModulus>()); } +}; - // i < 10**16, in this case 8+8 - if (i < 10'000'000'000'000'000ull) { - buffer = EncodeFullU32(static_cast<uint32_t>(div08), buffer); - little_endian::Store64(buffer, mod_result); - buffer += 8; - goto set_last_zero; - } else { - // 4 + 8 + 8 - uint64_t div016 = i / 10'000'000'000'000'000ull; - buffer = EncodeTenThousand(static_cast<uint32_t>(div016), buffer); - uint64_t mid_result = div08 - div016 * 100'000'000ull; - mid_result = PrepareTenThousands(mid_result / 10000, mid_result % 10000) + - kEightZeroBytes; - little_endian::Store64(buffer, mid_result); - buffer += 8; - little_endian::Store64(buffer, mod_result); - buffer += 8; - goto set_last_zero; +// Returns an iterator to the start of the suffix that was appended +template <typename T, typename BackwardIt> +std::enable_if_t<std::is_unsigned<T>::value, BackwardIt> +DoFastIntToBufferBackward(T v, BackwardIt end, uint32_t digits) { + using PromotedT = std::decay_t<decltype(+v)>; + using Converter = FastUIntToStringConverter<PromotedT, BackwardIt>; + (void)digits; + return Converter().FastIntToBufferBackward(v, end); +} + +template <typename T, typename BackwardIt> +std::enable_if_t<std::is_signed<T>::value, BackwardIt> +DoFastIntToBufferBackward(T v, BackwardIt end, uint32_t digits) { + if (absl::numbers_internal::IsNegative(v)) { + // Store the minus sign *before* we produce the number itself, not after. + // This gets us a tail call. + end[-static_cast<ptrdiff_t>(digits) - 1] = '-'; } -set_last_zero: - *buffer = '\0'; + return DoFastIntToBufferBackward( + absl::numbers_internal::UnsignedAbsoluteValue(v), end, digits); +} + +template <class T> +std::enable_if_t<std::is_integral<T>::value, int> +GetNumDigitsOrNegativeIfNegativeImpl(T v) { + const auto /* either bool or std::false_type */ is_negative = + absl::numbers_internal::IsNegative(v); + const int digits = static_cast<int>(absl::numbers_internal::Base10Digits( + absl::numbers_internal::UnsignedAbsoluteValue(v))); + return is_negative ? ~digits : digits; +} + +} // namespace + +void numbers_internal::PutTwoDigits(uint32_t i, absl::Nonnull<char*> buf) { + little_endian::Store16( + buf, static_cast<uint16_t>(PrepareTwoDigits(i) + kTwoZeroBytes)); +} + +absl::Nonnull<char*> numbers_internal::FastIntToBuffer( + uint32_t i, absl::Nonnull<char*> buffer) { + const uint32_t digits = absl::numbers_internal::Base10Digits(i); + buffer += digits; + *buffer = '\0'; // We're going backward, so store this first + FastIntToBufferBackward(i, buffer, digits); return buffer; } -char* numbers_internal::FastIntToBuffer(int64_t i, char* buffer) { - uint64_t u = static_cast<uint64_t>(i); - if (i < 0) { - *buffer++ = '-'; - u = 0 - u; - } - return numbers_internal::FastIntToBuffer(u, buffer); +absl::Nonnull<char*> numbers_internal::FastIntToBuffer( + int32_t i, absl::Nonnull<char*> buffer) { + buffer += static_cast<int>(i < 0); + uint32_t digits = absl::numbers_internal::Base10Digits( + absl::numbers_internal::UnsignedAbsoluteValue(i)); + buffer += digits; + *buffer = '\0'; // We're going backward, so store this first + FastIntToBufferBackward(i, buffer, digits); + return buffer; +} + +absl::Nonnull<char*> numbers_internal::FastIntToBuffer( + uint64_t i, absl::Nonnull<char*> buffer) { + uint32_t digits = absl::numbers_internal::Base10Digits(i); + buffer += digits; + *buffer = '\0'; // We're going backward, so store this first + FastIntToBufferBackward(i, buffer, digits); + return buffer; +} + +absl::Nonnull<char*> numbers_internal::FastIntToBuffer( + int64_t i, absl::Nonnull<char*> buffer) { + buffer += static_cast<int>(i < 0); + uint32_t digits = absl::numbers_internal::Base10Digits( + absl::numbers_internal::UnsignedAbsoluteValue(i)); + buffer += digits; + *buffer = '\0'; // We're going backward, so store this first + FastIntToBufferBackward(i, buffer, digits); + return buffer; +} + +absl::Nonnull<char*> numbers_internal::FastIntToBufferBackward( + uint32_t i, absl::Nonnull<char*> buffer_end, uint32_t exact_digit_count) { + return DoFastIntToBufferBackward(i, buffer_end, exact_digit_count); +} + +absl::Nonnull<char*> numbers_internal::FastIntToBufferBackward( + int32_t i, absl::Nonnull<char*> buffer_end, uint32_t exact_digit_count) { + return DoFastIntToBufferBackward(i, buffer_end, exact_digit_count); +} + +absl::Nonnull<char*> numbers_internal::FastIntToBufferBackward( + uint64_t i, absl::Nonnull<char*> buffer_end, uint32_t exact_digit_count) { + return DoFastIntToBufferBackward(i, buffer_end, exact_digit_count); +} + +absl::Nonnull<char*> numbers_internal::FastIntToBufferBackward( + int64_t i, absl::Nonnull<char*> buffer_end, uint32_t exact_digit_count) { + return DoFastIntToBufferBackward(i, buffer_end, exact_digit_count); +} + +int numbers_internal::GetNumDigitsOrNegativeIfNegative(signed char v) { + return GetNumDigitsOrNegativeIfNegativeImpl(v); +} +int numbers_internal::GetNumDigitsOrNegativeIfNegative(unsigned char v) { + return GetNumDigitsOrNegativeIfNegativeImpl(v); +} +int numbers_internal::GetNumDigitsOrNegativeIfNegative(short v) { // NOLINT + return GetNumDigitsOrNegativeIfNegativeImpl(v); +} +int numbers_internal::GetNumDigitsOrNegativeIfNegative( + unsigned short v) { // NOLINT + return GetNumDigitsOrNegativeIfNegativeImpl(v); +} +int numbers_internal::GetNumDigitsOrNegativeIfNegative(int v) { + return GetNumDigitsOrNegativeIfNegativeImpl(v); +} +int numbers_internal::GetNumDigitsOrNegativeIfNegative(unsigned int v) { + return GetNumDigitsOrNegativeIfNegativeImpl(v); +} +int numbers_internal::GetNumDigitsOrNegativeIfNegative(long v) { // NOLINT + return GetNumDigitsOrNegativeIfNegativeImpl(v); +} +int numbers_internal::GetNumDigitsOrNegativeIfNegative( + unsigned long v) { // NOLINT + return GetNumDigitsOrNegativeIfNegativeImpl(v); +} +int numbers_internal::GetNumDigitsOrNegativeIfNegative(long long v) { // NOLINT + return GetNumDigitsOrNegativeIfNegativeImpl(v); +} +int numbers_internal::GetNumDigitsOrNegativeIfNegative( + unsigned long long v) { // NOLINT + return GetNumDigitsOrNegativeIfNegativeImpl(v); } // Given a 128-bit number expressed as a pair of uint64_t, high half first, @@ -534,7 +769,8 @@ static ExpDigits SplitToSix(const double value) { // Helper function for fast formatting of floating-point. // The result is the same as "%g", a.k.a. "%.6g". -size_t numbers_internal::SixDigitsToBuffer(double d, char* const buffer) { +size_t numbers_internal::SixDigitsToBuffer(double d, + absl::Nonnull<char*> const buffer) { static_assert(std::numeric_limits<float>::is_iec559, "IEEE-754/IEC-559 support only"); @@ -681,9 +917,10 @@ static const int8_t kAsciiToInt[256] = { 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36}; // Parse the sign and optional hex or oct prefix in text. -inline bool safe_parse_sign_and_base(absl::string_view* text /*inout*/, - int* base_ptr /*inout*/, - bool* negative_ptr /*output*/) { +inline bool safe_parse_sign_and_base( + absl::Nonnull<absl::string_view*> text /*inout*/, + absl::Nonnull<int*> base_ptr /*inout*/, + absl::Nonnull<bool*> negative_ptr /*output*/) { if (text->data() == nullptr) { return false; } @@ -968,7 +1205,7 @@ ABSL_CONST_INIT const IntType LookupTables<IntType>::kVminOverBase[] = template <typename IntType> inline bool safe_parse_positive_int(absl::string_view text, int base, - IntType* value_p) { + absl::Nonnull<IntType*> value_p) { IntType value = 0; const IntType vmax = std::numeric_limits<IntType>::max(); assert(vmax > 0); @@ -1005,7 +1242,7 @@ inline bool safe_parse_positive_int(absl::string_view text, int base, template <typename IntType> inline bool safe_parse_negative_int(absl::string_view text, int base, - IntType* value_p) { + absl::Nonnull<IntType*> value_p) { IntType value = 0; const IntType vmin = std::numeric_limits<IntType>::min(); assert(vmin < 0); @@ -1049,8 +1286,8 @@ inline bool safe_parse_negative_int(absl::string_view text, int base, // Input format based on POSIX.1-2008 strtol // http://pubs.opengroup.org/onlinepubs/9699919799/functions/strtol.html template <typename IntType> -inline bool safe_int_internal(absl::string_view text, IntType* value_p, - int base) { +inline bool safe_int_internal(absl::string_view text, + absl::Nonnull<IntType*> value_p, int base) { *value_p = 0; bool negative; if (!safe_parse_sign_and_base(&text, &base, &negative)) { @@ -1064,8 +1301,8 @@ inline bool safe_int_internal(absl::string_view text, IntType* value_p, } template <typename IntType> -inline bool safe_uint_internal(absl::string_view text, IntType* value_p, - int base) { +inline bool safe_uint_internal(absl::string_view text, + absl::Nonnull<IntType*> value_p, int base) { *value_p = 0; bool negative; if (!safe_parse_sign_and_base(&text, &base, &negative) || negative) { @@ -1099,27 +1336,33 @@ ABSL_CONST_INIT ABSL_DLL const char kHexTable[513] = "e0e1e2e3e4e5e6e7e8e9eaebecedeeef" "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"; -bool safe_strto32_base(absl::string_view text, int32_t* value, int base) { +bool safe_strto32_base(absl::string_view text, absl::Nonnull<int32_t*> value, + int base) { return safe_int_internal<int32_t>(text, value, base); } -bool safe_strto64_base(absl::string_view text, int64_t* value, int base) { +bool safe_strto64_base(absl::string_view text, absl::Nonnull<int64_t*> value, + int base) { return safe_int_internal<int64_t>(text, value, base); } -bool safe_strto128_base(absl::string_view text, int128* value, int base) { +bool safe_strto128_base(absl::string_view text, absl::Nonnull<int128*> value, + int base) { return safe_int_internal<absl::int128>(text, value, base); } -bool safe_strtou32_base(absl::string_view text, uint32_t* value, int base) { +bool safe_strtou32_base(absl::string_view text, absl::Nonnull<uint32_t*> value, + int base) { return safe_uint_internal<uint32_t>(text, value, base); } -bool safe_strtou64_base(absl::string_view text, uint64_t* value, int base) { +bool safe_strtou64_base(absl::string_view text, absl::Nonnull<uint64_t*> value, + int base) { return safe_uint_internal<uint64_t>(text, value, base); } -bool safe_strtou128_base(absl::string_view text, uint128* value, int base) { +bool safe_strtou128_base(absl::string_view text, absl::Nonnull<uint128*> value, + int base) { return safe_uint_internal<absl::uint128>(text, value, base); } diff --git a/contrib/restricted/abseil-cpp/absl/strings/numbers.h b/contrib/restricted/abseil-cpp/absl/strings/numbers.h index d7630cef9c..ad4e66b634 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/numbers.h +++ b/contrib/restricted/abseil-cpp/absl/strings/numbers.h @@ -32,6 +32,7 @@ #endif #include <cstddef> +#include <cstdint> #include <cstdlib> #include <cstring> #include <ctime> @@ -39,9 +40,12 @@ #include <string> #include <type_traits> +#include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/internal/endian.h" #include "absl/base/macros.h" +#include "absl/base/nullability.h" +#include "absl/base/optimization.h" #include "absl/base/port.h" #include "absl/numeric/bits.h" #include "absl/numeric/int128.h" @@ -59,7 +63,8 @@ ABSL_NAMESPACE_BEGIN // encountered, this function returns `false`, leaving `out` in an unspecified // state. template <typename int_type> -ABSL_MUST_USE_RESULT bool SimpleAtoi(absl::string_view str, int_type* out); +ABSL_MUST_USE_RESULT bool SimpleAtoi(absl::string_view str, + absl::Nonnull<int_type*> out); // SimpleAtof() // @@ -70,7 +75,8 @@ ABSL_MUST_USE_RESULT bool SimpleAtoi(absl::string_view str, int_type* out); // allowed formats for `str`, except SimpleAtof() is locale-independent and will // always use the "C" locale. If any errors are encountered, this function // returns `false`, leaving `out` in an unspecified state. -ABSL_MUST_USE_RESULT bool SimpleAtof(absl::string_view str, float* out); +ABSL_MUST_USE_RESULT bool SimpleAtof(absl::string_view str, + absl::Nonnull<float*> out); // SimpleAtod() // @@ -81,7 +87,8 @@ ABSL_MUST_USE_RESULT bool SimpleAtof(absl::string_view str, float* out); // allowed formats for `str`, except SimpleAtod is locale-independent and will // always use the "C" locale. If any errors are encountered, this function // returns `false`, leaving `out` in an unspecified state. -ABSL_MUST_USE_RESULT bool SimpleAtod(absl::string_view str, double* out); +ABSL_MUST_USE_RESULT bool SimpleAtod(absl::string_view str, + absl::Nonnull<double*> out); // SimpleAtob() // @@ -91,7 +98,8 @@ ABSL_MUST_USE_RESULT bool SimpleAtod(absl::string_view str, double* out); // are interpreted as boolean `false`: "false", "f", "no", "n", "0". If any // errors are encountered, this function returns `false`, leaving `out` in an // unspecified state. -ABSL_MUST_USE_RESULT bool SimpleAtob(absl::string_view str, bool* out); +ABSL_MUST_USE_RESULT bool SimpleAtob(absl::string_view str, + absl::Nonnull<bool*> out); // SimpleHexAtoi() // @@ -104,13 +112,14 @@ ABSL_MUST_USE_RESULT bool SimpleAtob(absl::string_view str, bool* out); // by this function. If any errors are encountered, this function returns // `false`, leaving `out` in an unspecified state. template <typename int_type> -ABSL_MUST_USE_RESULT bool SimpleHexAtoi(absl::string_view str, int_type* out); +ABSL_MUST_USE_RESULT bool SimpleHexAtoi(absl::string_view str, + absl::Nonnull<int_type*> out); // Overloads of SimpleHexAtoi() for 128 bit integers. -ABSL_MUST_USE_RESULT inline bool SimpleHexAtoi(absl::string_view str, - absl::int128* out); -ABSL_MUST_USE_RESULT inline bool SimpleHexAtoi(absl::string_view str, - absl::uint128* out); +ABSL_MUST_USE_RESULT inline bool SimpleHexAtoi( + absl::string_view str, absl::Nonnull<absl::int128*> out); +ABSL_MUST_USE_RESULT inline bool SimpleHexAtoi( + absl::string_view str, absl::Nonnull<absl::uint128*> out); ABSL_NAMESPACE_END } // namespace absl @@ -132,42 +141,136 @@ ABSL_DLL extern const char // PutTwoDigits(42, buf); // // buf[0] == '4' // // buf[1] == '2' -void PutTwoDigits(uint32_t i, char* buf); +void PutTwoDigits(uint32_t i, absl::Nonnull<char*> buf); // safe_strto?() functions for implementing SimpleAtoi() -bool safe_strto32_base(absl::string_view text, int32_t* value, int base); -bool safe_strto64_base(absl::string_view text, int64_t* value, int base); -bool safe_strto128_base(absl::string_view text, absl::int128* value, - int base); -bool safe_strtou32_base(absl::string_view text, uint32_t* value, int base); -bool safe_strtou64_base(absl::string_view text, uint64_t* value, int base); -bool safe_strtou128_base(absl::string_view text, absl::uint128* value, - int base); +bool safe_strto32_base(absl::string_view text, absl::Nonnull<int32_t*> value, + int base); +bool safe_strto64_base(absl::string_view text, absl::Nonnull<int64_t*> value, + int base); +bool safe_strto128_base(absl::string_view text, + absl::Nonnull<absl::int128*> value, int base); +bool safe_strtou32_base(absl::string_view text, absl::Nonnull<uint32_t*> value, + int base); +bool safe_strtou64_base(absl::string_view text, absl::Nonnull<uint64_t*> value, + int base); +bool safe_strtou128_base(absl::string_view text, + absl::Nonnull<absl::uint128*> value, int base); static const int kFastToBufferSize = 32; static const int kSixDigitsToBufferSize = 16; +template <class T> +std::enable_if_t<!std::is_unsigned<T>::value, bool> IsNegative(const T& v) { + return v < T(); +} + +template <class T> +std::enable_if_t<std::is_unsigned<T>::value, std::false_type> IsNegative( + const T&) { + // The integer is unsigned, so return a compile-time constant. + // This can help the optimizer avoid having to prove bool to be false later. + return std::false_type(); +} + +template <class T> +std::enable_if_t<std::is_unsigned<std::decay_t<T>>::value, T&&> +UnsignedAbsoluteValue(T&& v ABSL_ATTRIBUTE_LIFETIME_BOUND) { + // The value is unsigned; just return the original. + return std::forward<T>(v); +} + +template <class T> +ABSL_ATTRIBUTE_CONST_FUNCTION + std::enable_if_t<!std::is_unsigned<T>::value, std::make_unsigned_t<T>> + UnsignedAbsoluteValue(T v) { + using U = std::make_unsigned_t<T>; + return IsNegative(v) ? U() - static_cast<U>(v) : static_cast<U>(v); +} + +// Returns the number of base-10 digits in the given number. +// Note that this strictly counts digits. It does not count the sign. +// The `initial_digits` parameter is the starting point, which is normally equal +// to 1 because the number of digits in 0 is 1 (a special case). +// However, callers may e.g. wish to change it to 2 to account for the sign. +template <typename T> +std::enable_if_t<std::is_unsigned<T>::value, uint32_t> Base10Digits( + T v, const uint32_t initial_digits = 1) { + uint32_t r = initial_digits; + // If code size becomes an issue, the 'if' stage can be removed for a minor + // performance loss. + for (;;) { + if (ABSL_PREDICT_TRUE(v < 10 * 10)) { + r += (v >= 10); + break; + } + if (ABSL_PREDICT_TRUE(v < 1000 * 10)) { + r += (v >= 1000) + 2; + break; + } + if (ABSL_PREDICT_TRUE(v < 100000 * 10)) { + r += (v >= 100000) + 4; + break; + } + r += 6; + v = static_cast<T>(v / 1000000); + } + return r; +} + +template <typename T> +std::enable_if_t<std::is_signed<T>::value, uint32_t> Base10Digits( + T v, uint32_t r = 1) { + // Branchlessly add 1 to account for a minus sign. + r += static_cast<uint32_t>(IsNegative(v)); + return Base10Digits(UnsignedAbsoluteValue(v), r); +} + +// These functions return the number of base-10 digits, but multiplied by -1 if +// the input itself is negative. This is handy and efficient for later usage, +// since the bitwise complement of the result becomes equal to the number of +// characters required. +ABSL_ATTRIBUTE_CONST_FUNCTION int GetNumDigitsOrNegativeIfNegative( + signed char v); +ABSL_ATTRIBUTE_CONST_FUNCTION int GetNumDigitsOrNegativeIfNegative( + unsigned char v); +ABSL_ATTRIBUTE_CONST_FUNCTION int GetNumDigitsOrNegativeIfNegative( + short v); // NOLINT +ABSL_ATTRIBUTE_CONST_FUNCTION int GetNumDigitsOrNegativeIfNegative( + unsigned short v); // NOLINT +ABSL_ATTRIBUTE_CONST_FUNCTION int GetNumDigitsOrNegativeIfNegative(int v); +ABSL_ATTRIBUTE_CONST_FUNCTION int GetNumDigitsOrNegativeIfNegative( + unsigned int v); +ABSL_ATTRIBUTE_CONST_FUNCTION int GetNumDigitsOrNegativeIfNegative( + long v); // NOLINT +ABSL_ATTRIBUTE_CONST_FUNCTION int GetNumDigitsOrNegativeIfNegative( + unsigned long v); // NOLINT +ABSL_ATTRIBUTE_CONST_FUNCTION int GetNumDigitsOrNegativeIfNegative( + long long v); // NOLINT +ABSL_ATTRIBUTE_CONST_FUNCTION int GetNumDigitsOrNegativeIfNegative( + unsigned long long v); // NOLINT + // Helper function for fast formatting of floating-point values. // The result is the same as printf's "%g", a.k.a. "%.6g"; that is, six // significant digits are returned, trailing zeros are removed, and numbers // outside the range 0.0001-999999 are output using scientific notation // (1.23456e+06). This routine is heavily optimized. // Required buffer size is `kSixDigitsToBufferSize`. -size_t SixDigitsToBuffer(double d, char* buffer); +size_t SixDigitsToBuffer(double d, absl::Nonnull<char*> buffer); -// These functions are intended for speed. All functions take an output buffer +// All of these functions take an output buffer // as an argument and return a pointer to the last byte they wrote, which is the // terminating '\0'. At most `kFastToBufferSize` bytes are written. -char* FastIntToBuffer(int32_t, char*); -char* FastIntToBuffer(uint32_t, char*); -char* FastIntToBuffer(int64_t, char*); -char* FastIntToBuffer(uint64_t, char*); +absl::Nonnull<char*> FastIntToBuffer(int32_t i, absl::Nonnull<char*> buffer); +absl::Nonnull<char*> FastIntToBuffer(uint32_t i, absl::Nonnull<char*> buffer); +absl::Nonnull<char*> FastIntToBuffer(int64_t i, absl::Nonnull<char*> buffer); +absl::Nonnull<char*> FastIntToBuffer(uint64_t i, absl::Nonnull<char*> buffer); // For enums and integer types that are not an exact match for the types above, // use templates to call the appropriate one of the four overloads above. template <typename int_type> -char* FastIntToBuffer(int_type i, char* buffer) { +absl::Nonnull<char*> FastIntToBuffer(int_type i, absl::Nonnull<char*> buffer) { static_assert(sizeof(i) <= 64 / 8, "FastIntToBuffer works only with 64-bit-or-less integers."); // TODO(jorg): This signed-ness check is used because it works correctly @@ -191,10 +294,63 @@ char* FastIntToBuffer(int_type i, char* buffer) { } } +// These functions do NOT add any null-terminator. +// They return a pointer to the beginning of the written string. +// The digit counts provided must *exactly* match the number of base-10 digits +// in the number, or the behavior is undefined. +// (i.e. do NOT count the minus sign, or over- or under-count the digits.) +absl::Nonnull<char*> FastIntToBufferBackward(int32_t i, + absl::Nonnull<char*> buffer_end, + uint32_t exact_digit_count); +absl::Nonnull<char*> FastIntToBufferBackward(uint32_t i, + absl::Nonnull<char*> buffer_end, + uint32_t exact_digit_count); +absl::Nonnull<char*> FastIntToBufferBackward(int64_t i, + absl::Nonnull<char*> buffer_end, + uint32_t exact_digit_count); +absl::Nonnull<char*> FastIntToBufferBackward(uint64_t i, + absl::Nonnull<char*> buffer_end, + uint32_t exact_digit_count); + +// For enums and integer types that are not an exact match for the types above, +// use templates to call the appropriate one of the four overloads above. +template <typename int_type> +absl::Nonnull<char*> FastIntToBufferBackward(int_type i, + absl::Nonnull<char*> buffer_end, + uint32_t exact_digit_count) { + static_assert( + sizeof(i) <= 64 / 8, + "FastIntToBufferBackward works only with 64-bit-or-less integers."); + // This signed-ness check is used because it works correctly + // with enums, and it also serves to check that int_type is not a pointer. + // If one day something like std::is_signed<enum E> works, switch to it. + // These conditions are constexpr bools to suppress MSVC warning C4127. + constexpr bool kIsSigned = static_cast<int_type>(1) - 2 < 0; + constexpr bool kUse64Bit = sizeof(i) > 32 / 8; + if (kIsSigned) { + if (kUse64Bit) { + return FastIntToBufferBackward(static_cast<int64_t>(i), buffer_end, + exact_digit_count); + } else { + return FastIntToBufferBackward(static_cast<int32_t>(i), buffer_end, + exact_digit_count); + } + } else { + if (kUse64Bit) { + return FastIntToBufferBackward(static_cast<uint64_t>(i), buffer_end, + exact_digit_count); + } else { + return FastIntToBufferBackward(static_cast<uint32_t>(i), buffer_end, + exact_digit_count); + } + } +} + // Implementation of SimpleAtoi, generalized to support arbitrary base (used // with base different from 10 elsewhere in Abseil implementation). template <typename int_type> -ABSL_MUST_USE_RESULT bool safe_strtoi_base(absl::string_view s, int_type* out, +ABSL_MUST_USE_RESULT bool safe_strtoi_base(absl::string_view s, + absl::Nonnull<int_type*> out, int base) { static_assert(sizeof(*out) == 4 || sizeof(*out) == 8, "SimpleAtoi works only with 32-bit or 64-bit integers."); @@ -237,7 +393,7 @@ ABSL_MUST_USE_RESULT bool safe_strtoi_base(absl::string_view s, int_type* out, // without the terminating null character. Thus `out` must be of length >= 16. // Returns the number of non-pad digits of the output (it can never be zero // since 0 has one digit). -inline size_t FastHexToBufferZeroPad16(uint64_t val, char* out) { +inline size_t FastHexToBufferZeroPad16(uint64_t val, absl::Nonnull<char*> out) { #ifdef ABSL_INTERNAL_HAVE_SSSE3 uint64_t be = absl::big_endian::FromHost64(val); const auto kNibbleMask = _mm_set1_epi8(0xf); @@ -263,32 +419,34 @@ inline size_t FastHexToBufferZeroPad16(uint64_t val, char* out) { } // namespace numbers_internal template <typename int_type> -ABSL_MUST_USE_RESULT bool SimpleAtoi(absl::string_view str, int_type* out) { +ABSL_MUST_USE_RESULT bool SimpleAtoi(absl::string_view str, + absl::Nonnull<int_type*> out) { return numbers_internal::safe_strtoi_base(str, out, 10); } ABSL_MUST_USE_RESULT inline bool SimpleAtoi(absl::string_view str, - absl::int128* out) { + absl::Nonnull<absl::int128*> out) { return numbers_internal::safe_strto128_base(str, out, 10); } ABSL_MUST_USE_RESULT inline bool SimpleAtoi(absl::string_view str, - absl::uint128* out) { + absl::Nonnull<absl::uint128*> out) { return numbers_internal::safe_strtou128_base(str, out, 10); } template <typename int_type> -ABSL_MUST_USE_RESULT bool SimpleHexAtoi(absl::string_view str, int_type* out) { +ABSL_MUST_USE_RESULT bool SimpleHexAtoi(absl::string_view str, + absl::Nonnull<int_type*> out) { return numbers_internal::safe_strtoi_base(str, out, 16); } -ABSL_MUST_USE_RESULT inline bool SimpleHexAtoi(absl::string_view str, - absl::int128* out) { +ABSL_MUST_USE_RESULT inline bool SimpleHexAtoi( + absl::string_view str, absl::Nonnull<absl::int128*> out) { return numbers_internal::safe_strto128_base(str, out, 16); } -ABSL_MUST_USE_RESULT inline bool SimpleHexAtoi(absl::string_view str, - absl::uint128* out) { +ABSL_MUST_USE_RESULT inline bool SimpleHexAtoi( + absl::string_view str, absl::Nonnull<absl::uint128*> out) { return numbers_internal::safe_strtou128_base(str, out, 16); } diff --git a/contrib/restricted/abseil-cpp/absl/strings/str_cat.cc b/contrib/restricted/abseil-cpp/absl/strings/str_cat.cc index 2e49c31b68..098ab18388 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/str_cat.cc +++ b/contrib/restricted/abseil-cpp/absl/strings/str_cat.cc @@ -16,13 +16,15 @@ #include <assert.h> -#include <algorithm> #include <cstddef> #include <cstdint> #include <cstring> +#include <initializer_list> #include <string> +#include <type_traits> -#include "absl/strings/ascii.h" +#include "absl/base/config.h" +#include "absl/base/nullability.h" #include "absl/strings/internal/resize_uninitialized.h" #include "absl/strings/numbers.h" #include "absl/strings/string_view.h" @@ -30,6 +32,7 @@ namespace absl { ABSL_NAMESPACE_BEGIN + // ---------------------------------------------------------------------- // StrCat() // This merges the given strings or integers, with no delimiter. This @@ -37,9 +40,10 @@ ABSL_NAMESPACE_BEGIN // of a mix of raw C strings, string_views, strings, and integer values. // ---------------------------------------------------------------------- +namespace { // Append is merely a version of memcpy that returns the address of the byte // after the area just overwritten. -static char* Append(char* out, const AlphaNum& x) { +absl::Nonnull<char*> Append(absl::Nonnull<char*> out, const AlphaNum& x) { // memcpy is allowed to overwrite arbitrary memory, so doing this after the // call would force an extra fetch of x.size(). char* after = out + x.size(); @@ -49,6 +53,8 @@ static char* Append(char* out, const AlphaNum& x) { return after; } +} // namespace + std::string StrCat(const AlphaNum& a, const AlphaNum& b) { std::string result; absl::strings_internal::STLStringResizeUninitialized(&result, @@ -92,6 +98,130 @@ std::string StrCat(const AlphaNum& a, const AlphaNum& b, const AlphaNum& c, namespace strings_internal { // Do not call directly - these are not part of the public API. +void STLStringAppendUninitializedAmortized(std::string* dest, + size_t to_append) { + strings_internal::AppendUninitializedTraits<std::string>::Append(dest, + to_append); +} + +template <typename Integer> +std::enable_if_t<std::is_integral<Integer>::value, std::string> IntegerToString( + Integer i) { + std::string str; + const auto /* either bool or std::false_type */ is_negative = + absl::numbers_internal::IsNegative(i); + const uint32_t digits = absl::numbers_internal::Base10Digits( + absl::numbers_internal::UnsignedAbsoluteValue(i)); + absl::strings_internal::STLStringResizeUninitialized( + &str, digits + static_cast<uint32_t>(is_negative)); + absl::numbers_internal::FastIntToBufferBackward(i, &str[str.size()], digits); + return str; +} + +template <> +std::string IntegerToString(long i) { // NOLINT + if (sizeof(i) <= sizeof(int)) { + return IntegerToString(static_cast<int>(i)); + } else { + return IntegerToString(static_cast<long long>(i)); // NOLINT + } +} + +template <> +std::string IntegerToString(unsigned long i) { // NOLINT + if (sizeof(i) <= sizeof(unsigned int)) { + return IntegerToString(static_cast<unsigned int>(i)); + } else { + return IntegerToString(static_cast<unsigned long long>(i)); // NOLINT + } +} + +template <typename Float> +std::enable_if_t<std::is_floating_point<Float>::value, std::string> +FloatToString(Float f) { + std::string result; + strings_internal::STLStringResizeUninitialized( + &result, numbers_internal::kSixDigitsToBufferSize); + char* start = &result[0]; + result.erase(numbers_internal::SixDigitsToBuffer(f, start)); + return result; +} + +std::string SingleArgStrCat(int x) { return IntegerToString(x); } +std::string SingleArgStrCat(unsigned int x) { return IntegerToString(x); } +// NOLINTNEXTLINE +std::string SingleArgStrCat(long x) { return IntegerToString(x); } +// NOLINTNEXTLINE +std::string SingleArgStrCat(unsigned long x) { return IntegerToString(x); } +// NOLINTNEXTLINE +std::string SingleArgStrCat(long long x) { return IntegerToString(x); } +// NOLINTNEXTLINE +std::string SingleArgStrCat(unsigned long long x) { return IntegerToString(x); } +std::string SingleArgStrCat(float x) { return FloatToString(x); } +std::string SingleArgStrCat(double x) { return FloatToString(x); } + +template <class Integer> +std::enable_if_t<std::is_integral<Integer>::value, void> AppendIntegerToString( + std::string& str, Integer i) { + const auto /* either bool or std::false_type */ is_negative = + absl::numbers_internal::IsNegative(i); + const uint32_t digits = absl::numbers_internal::Base10Digits( + absl::numbers_internal::UnsignedAbsoluteValue(i)); + absl::strings_internal::STLStringAppendUninitializedAmortized( + &str, digits + static_cast<uint32_t>(is_negative)); + absl::numbers_internal::FastIntToBufferBackward(i, &str[str.size()], digits); +} + +template <> +void AppendIntegerToString(std::string& str, long i) { // NOLINT + if (sizeof(i) <= sizeof(int)) { + return AppendIntegerToString(str, static_cast<int>(i)); + } else { + return AppendIntegerToString(str, static_cast<long long>(i)); // NOLINT + } +} + +template <> +void AppendIntegerToString(std::string& str, + unsigned long i) { // NOLINT + if (sizeof(i) <= sizeof(unsigned int)) { + return AppendIntegerToString(str, static_cast<unsigned int>(i)); + } else { + return AppendIntegerToString(str, + static_cast<unsigned long long>(i)); // NOLINT + } +} + +// `SingleArgStrAppend` overloads are defined here for the same reasons as with +// `SingleArgStrCat` above. +void SingleArgStrAppend(std::string& str, int x) { + return AppendIntegerToString(str, x); +} + +void SingleArgStrAppend(std::string& str, unsigned int x) { + return AppendIntegerToString(str, x); +} + +// NOLINTNEXTLINE +void SingleArgStrAppend(std::string& str, long x) { + return AppendIntegerToString(str, x); +} + +// NOLINTNEXTLINE +void SingleArgStrAppend(std::string& str, unsigned long x) { + return AppendIntegerToString(str, x); +} + +// NOLINTNEXTLINE +void SingleArgStrAppend(std::string& str, long long x) { + return AppendIntegerToString(str, x); +} + +// NOLINTNEXTLINE +void SingleArgStrAppend(std::string& str, unsigned long long x) { + return AppendIntegerToString(str, x); +} + std::string CatPieces(std::initializer_list<absl::string_view> pieces) { std::string result; size_t total_size = 0; @@ -120,15 +250,15 @@ std::string CatPieces(std::initializer_list<absl::string_view> pieces) { assert(((src).size() == 0) || \ (uintptr_t((src).data() - (dest).data()) > uintptr_t((dest).size()))) -void AppendPieces(std::string* dest, +void AppendPieces(absl::Nonnull<std::string*> dest, std::initializer_list<absl::string_view> pieces) { size_t old_size = dest->size(); - size_t total_size = old_size; + size_t to_append = 0; for (absl::string_view piece : pieces) { ASSERT_NO_OVERLAP(*dest, piece); - total_size += piece.size(); + to_append += piece.size(); } - strings_internal::STLStringResizeUninitializedAmortized(dest, total_size); + strings_internal::STLStringAppendUninitializedAmortized(dest, to_append); char* const begin = &(*dest)[0]; char* out = begin + old_size; @@ -144,23 +274,23 @@ void AppendPieces(std::string* dest, } // namespace strings_internal -void StrAppend(std::string* dest, const AlphaNum& a) { +void StrAppend(absl::Nonnull<std::string*> dest, const AlphaNum& a) { ASSERT_NO_OVERLAP(*dest, a); std::string::size_type old_size = dest->size(); - strings_internal::STLStringResizeUninitializedAmortized(dest, - old_size + a.size()); + strings_internal::STLStringAppendUninitializedAmortized(dest, a.size()); char* const begin = &(*dest)[0]; char* out = begin + old_size; out = Append(out, a); assert(out == begin + dest->size()); } -void StrAppend(std::string* dest, const AlphaNum& a, const AlphaNum& b) { +void StrAppend(absl::Nonnull<std::string*> dest, const AlphaNum& a, + const AlphaNum& b) { ASSERT_NO_OVERLAP(*dest, a); ASSERT_NO_OVERLAP(*dest, b); std::string::size_type old_size = dest->size(); - strings_internal::STLStringResizeUninitializedAmortized( - dest, old_size + a.size() + b.size()); + strings_internal::STLStringAppendUninitializedAmortized(dest, + a.size() + b.size()); char* const begin = &(*dest)[0]; char* out = begin + old_size; out = Append(out, a); @@ -168,14 +298,14 @@ void StrAppend(std::string* dest, const AlphaNum& a, const AlphaNum& b) { assert(out == begin + dest->size()); } -void StrAppend(std::string* dest, const AlphaNum& a, const AlphaNum& b, - const AlphaNum& c) { +void StrAppend(absl::Nonnull<std::string*> dest, const AlphaNum& a, + const AlphaNum& b, const AlphaNum& c) { ASSERT_NO_OVERLAP(*dest, a); ASSERT_NO_OVERLAP(*dest, b); ASSERT_NO_OVERLAP(*dest, c); std::string::size_type old_size = dest->size(); - strings_internal::STLStringResizeUninitializedAmortized( - dest, old_size + a.size() + b.size() + c.size()); + strings_internal::STLStringAppendUninitializedAmortized( + dest, a.size() + b.size() + c.size()); char* const begin = &(*dest)[0]; char* out = begin + old_size; out = Append(out, a); @@ -184,15 +314,15 @@ void StrAppend(std::string* dest, const AlphaNum& a, const AlphaNum& b, assert(out == begin + dest->size()); } -void StrAppend(std::string* dest, const AlphaNum& a, const AlphaNum& b, - const AlphaNum& c, const AlphaNum& d) { +void StrAppend(absl::Nonnull<std::string*> dest, const AlphaNum& a, + const AlphaNum& b, const AlphaNum& c, const AlphaNum& d) { ASSERT_NO_OVERLAP(*dest, a); ASSERT_NO_OVERLAP(*dest, b); ASSERT_NO_OVERLAP(*dest, c); ASSERT_NO_OVERLAP(*dest, d); std::string::size_type old_size = dest->size(); - strings_internal::STLStringResizeUninitializedAmortized( - dest, old_size + a.size() + b.size() + c.size() + d.size()); + strings_internal::STLStringAppendUninitializedAmortized( + dest, a.size() + b.size() + c.size() + d.size()); char* const begin = &(*dest)[0]; char* out = begin + old_size; out = Append(out, a); diff --git a/contrib/restricted/abseil-cpp/absl/strings/str_cat.h b/contrib/restricted/abseil-cpp/absl/strings/str_cat.h index d5f71ff003..ea2c4dcad8 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/str_cat.h +++ b/contrib/restricted/abseil-cpp/absl/strings/str_cat.h @@ -89,6 +89,8 @@ #include <algorithm> #include <array> +#include <cassert> +#include <cstddef> #include <cstdint> #include <cstring> #include <string> @@ -97,8 +99,11 @@ #include <vector> #include "absl/base/attributes.h" +#include "absl/base/nullability.h" #include "absl/base/port.h" -#include "absl/strings/internal/has_absl_stringify.h" +#include "absl/meta/type_traits.h" +#include "absl/strings/has_absl_stringify.h" +#include "absl/strings/internal/resize_uninitialized.h" #include "absl/strings/internal/stringify_sink.h" #include "absl/strings/numbers.h" #include "absl/strings/string_view.h" @@ -201,7 +206,7 @@ struct Hex { !std::is_pointer<Int>::value>::type* = nullptr) : Hex(spec, static_cast<uint64_t>(v)) {} template <typename Pointee> - explicit Hex(Pointee* v, PadSpec spec = absl::kNoPad) + explicit Hex(absl::Nullable<Pointee*> v, PadSpec spec = absl::kNoPad) : Hex(spec, reinterpret_cast<uintptr_t>(v)) {} template <typename S> @@ -253,10 +258,9 @@ struct Dec { typename std::enable_if<(sizeof(Int) <= 8)>::type* = nullptr) : value(v >= 0 ? static_cast<uint64_t>(v) : uint64_t{0} - static_cast<uint64_t>(v)), - width(spec == absl::kNoPad - ? 1 - : spec >= absl::kSpacePad2 ? spec - absl::kSpacePad2 + 2 - : spec - absl::kZeroPad2 + 2), + width(spec == absl::kNoPad ? 1 + : spec >= absl::kSpacePad2 ? spec - absl::kSpacePad2 + 2 + : spec - absl::kZeroPad2 + 2), fill(spec >= absl::kSpacePad2 ? ' ' : '0'), neg(v < 0) {} @@ -344,7 +348,7 @@ class AlphaNum { ABSL_ATTRIBUTE_LIFETIME_BOUND) : piece_(&buf.data[0], buf.size) {} - AlphaNum(const char* c_str // NOLINT(runtime/explicit) + AlphaNum(absl::Nullable<const char*> c_str // NOLINT(runtime/explicit) ABSL_ATTRIBUTE_LIFETIME_BOUND) : piece_(NullSafeStringView(c_str)) {} AlphaNum(absl::string_view pc // NOLINT(runtime/explicit) @@ -352,7 +356,7 @@ class AlphaNum { : piece_(pc) {} template <typename T, typename = typename std::enable_if< - strings_internal::HasAbslStringify<T>::value>::type> + HasAbslStringify<T>::value>::type> AlphaNum( // NOLINT(runtime/explicit) const T& v ABSL_ATTRIBUTE_LIFETIME_BOUND, strings_internal::StringifySink&& sink ABSL_ATTRIBUTE_LIFETIME_BOUND = {}) @@ -371,7 +375,7 @@ class AlphaNum { AlphaNum& operator=(const AlphaNum&) = delete; absl::string_view::size_type size() const { return piece_.size(); } - const char* data() const { return piece_.data(); } + absl::Nullable<const char*> data() const { return piece_.data(); } absl::string_view Piece() const { return piece_; } // Match unscoped enums. Use integral promotion so that a `char`-backed @@ -379,17 +383,17 @@ class AlphaNum { template <typename T, typename = typename std::enable_if< std::is_enum<T>{} && std::is_convertible<T, int>{} && - !strings_internal::HasAbslStringify<T>::value>::type> + !HasAbslStringify<T>::value>::type> AlphaNum(T e) // NOLINT(runtime/explicit) : AlphaNum(+e) {} // This overload matches scoped enums. We must explicitly cast to the // underlying type, but use integral promotion for the same reason as above. template <typename T, - typename std::enable_if< - std::is_enum<T>{} && !std::is_convertible<T, int>{} && - !strings_internal::HasAbslStringify<T>::value, - char*>::type = nullptr> + typename std::enable_if<std::is_enum<T>{} && + !std::is_convertible<T, int>{} && + !HasAbslStringify<T>::value, + char*>::type = nullptr> AlphaNum(T e) // NOLINT(runtime/explicit) : AlphaNum(+static_cast<typename std::underlying_type<T>::type>(e)) {} @@ -441,13 +445,48 @@ namespace strings_internal { // Do not call directly - this is not part of the public API. std::string CatPieces(std::initializer_list<absl::string_view> pieces); -void AppendPieces(std::string* dest, +void AppendPieces(absl::Nonnull<std::string*> dest, std::initializer_list<absl::string_view> pieces); +void STLStringAppendUninitializedAmortized(std::string* dest, size_t to_append); + +// `SingleArgStrCat` overloads take built-in `int`, `long` and `long long` types +// (signed / unsigned) to avoid ambiguity on the call side. If we used int32_t +// and int64_t, then at least one of the three (`int` / `long` / `long long`) +// would have been ambiguous when passed to `SingleArgStrCat`. +std::string SingleArgStrCat(int x); +std::string SingleArgStrCat(unsigned int x); +std::string SingleArgStrCat(long x); // NOLINT +std::string SingleArgStrCat(unsigned long x); // NOLINT +std::string SingleArgStrCat(long long x); // NOLINT +std::string SingleArgStrCat(unsigned long long x); // NOLINT +std::string SingleArgStrCat(float x); +std::string SingleArgStrCat(double x); + +// `SingleArgStrAppend` overloads are defined here for the same reasons as with +// `SingleArgStrCat` above. +void SingleArgStrAppend(std::string& str, int x); +void SingleArgStrAppend(std::string& str, unsigned int x); +void SingleArgStrAppend(std::string& str, long x); // NOLINT +void SingleArgStrAppend(std::string& str, unsigned long x); // NOLINT +void SingleArgStrAppend(std::string& str, long long x); // NOLINT +void SingleArgStrAppend(std::string& str, unsigned long long x); // NOLINT + +template <typename T, + typename = std::enable_if_t<std::is_arithmetic<T>::value && + !std::is_same<T, char>::value && + !std::is_same<T, bool>::value>> +using EnableIfFastCase = T; + } // namespace strings_internal ABSL_MUST_USE_RESULT inline std::string StrCat() { return std::string(); } +template <typename T> +ABSL_MUST_USE_RESULT inline std::string StrCat( + strings_internal::EnableIfFastCase<T> a) { + return strings_internal::SingleArgStrCat(a); +} ABSL_MUST_USE_RESULT inline std::string StrCat(const AlphaNum& a) { return std::string(a.data(), a.size()); } @@ -495,24 +534,87 @@ ABSL_MUST_USE_RESULT inline std::string StrCat( // absl::string_view p = s; // StrAppend(&s, p); -inline void StrAppend(std::string*) {} -void StrAppend(std::string* dest, const AlphaNum& a); -void StrAppend(std::string* dest, const AlphaNum& a, const AlphaNum& b); -void StrAppend(std::string* dest, const AlphaNum& a, const AlphaNum& b, - const AlphaNum& c); -void StrAppend(std::string* dest, const AlphaNum& a, const AlphaNum& b, - const AlphaNum& c, const AlphaNum& d); +inline void StrAppend(absl::Nonnull<std::string*>) {} +void StrAppend(absl::Nonnull<std::string*> dest, const AlphaNum& a); +void StrAppend(absl::Nonnull<std::string*> dest, const AlphaNum& a, + const AlphaNum& b); +void StrAppend(absl::Nonnull<std::string*> dest, const AlphaNum& a, + const AlphaNum& b, const AlphaNum& c); +void StrAppend(absl::Nonnull<std::string*> dest, const AlphaNum& a, + const AlphaNum& b, const AlphaNum& c, const AlphaNum& d); // Support 5 or more arguments template <typename... AV> -inline void StrAppend(std::string* dest, const AlphaNum& a, const AlphaNum& b, - const AlphaNum& c, const AlphaNum& d, const AlphaNum& e, - const AV&... args) { +inline void StrAppend(absl::Nonnull<std::string*> dest, const AlphaNum& a, + const AlphaNum& b, const AlphaNum& c, const AlphaNum& d, + const AlphaNum& e, const AV&... args) { strings_internal::AppendPieces( dest, {a.Piece(), b.Piece(), c.Piece(), d.Piece(), e.Piece(), static_cast<const AlphaNum&>(args).Piece()...}); } +template <class String, class T> +std::enable_if_t< + std::is_integral<absl::strings_internal::EnableIfFastCase<T>>::value, void> +StrAppend(absl::Nonnull<String*> result, T i) { + return absl::strings_internal::SingleArgStrAppend(*result, i); +} + +// This overload is only selected if all the parameters are numbers that can be +// handled quickly. +// Later we can look into how we can extend this to more general argument +// mixtures without bloating codegen too much, or copying unnecessarily. +template <typename String, typename... T> +std::enable_if_t< + (sizeof...(T) > 1), + std::common_type_t<std::conditional_t< + true, void, absl::strings_internal::EnableIfFastCase<T>>...>> +StrAppend(absl::Nonnull<String*> str, T... args) { + // Do not add unnecessary variables, logic, or even "free" lambdas here. + // They can add overhead for the compiler and/or at run time. + // Furthermore, assume this function will be inlined. + // This function is carefully tailored to be able to be largely optimized away + // so that it becomes near-equivalent to the caller handling each argument + // individually while minimizing register pressure, so that the compiler + // can inline it with minimal overhead. + + // First, calculate the total length, so we can perform just a single resize. + // Save all the lengths for later. + size_t total_length = 0; + const ptrdiff_t lengths[] = { + absl::numbers_internal::GetNumDigitsOrNegativeIfNegative(args)...}; + for (const ptrdiff_t possibly_negative_length : lengths) { + // Lengths are negative for negative numbers. Keep them for later use, but + // take their absolute values for calculating total lengths; + total_length += possibly_negative_length < 0 + ? static_cast<size_t>(-possibly_negative_length) + : static_cast<size_t>(possibly_negative_length); + } + + // Now reserve space for all the arguments. + const size_t old_size = str->size(); + absl::strings_internal::STLStringAppendUninitializedAmortized(str, + total_length); + + // Finally, output each argument one-by-one, from left to right. + size_t i = 0; // The current argument we're processing + ptrdiff_t n; // The length of the current argument + typename String::pointer pos = &(*str)[old_size]; + using SomeTrivialEmptyType = std::false_type; + // Ugly code due to the lack of C++14 fold expression makes us. + const SomeTrivialEmptyType dummy1; + for (const SomeTrivialEmptyType& dummy2 : + {(/* Comma expressions are poor man's C++17 fold expression for C++14 */ + (void)(n = lengths[i]), + (void)(n < 0 ? (void)(*pos++ = '-'), (n = ~n) : 0), + (void)absl::numbers_internal::FastIntToBufferBackward( + absl::numbers_internal::UnsignedAbsoluteValue(std::move(args)), + pos += n, static_cast<uint32_t>(n)), + (void)++i, dummy1)...}) { + (void)dummy2; // Remove & migrate to fold expressions in C++17 + } +} + // Helper function for the future StrCat default floating-point format, %.6g // This is fast. inline strings_internal::AlphaNumBuffer< diff --git a/contrib/restricted/abseil-cpp/absl/strings/str_format.h b/contrib/restricted/abseil-cpp/absl/strings/str_format.h index 023e4350d3..66b6af5854 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/str_format.h +++ b/contrib/restricted/abseil-cpp/absl/strings/str_format.h @@ -72,14 +72,21 @@ #ifndef ABSL_STRINGS_STR_FORMAT_H_ #define ABSL_STRINGS_STR_FORMAT_H_ +#include <cstdint> #include <cstdio> #include <string> +#include <type_traits> +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/nullability.h" #include "absl/strings/internal/str_format/arg.h" // IWYU pragma: export #include "absl/strings/internal/str_format/bind.h" // IWYU pragma: export #include "absl/strings/internal/str_format/checker.h" // IWYU pragma: export #include "absl/strings/internal/str_format/extension.h" // IWYU pragma: export #include "absl/strings/internal/str_format/parser.h" // IWYU pragma: export +#include "absl/strings/string_view.h" +#include "absl/types/span.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -104,7 +111,8 @@ class UntypedFormatSpec { explicit UntypedFormatSpec(string_view s) : spec_(s) {} protected: - explicit UntypedFormatSpec(const str_format_internal::ParsedFormatBase* pc) + explicit UntypedFormatSpec( + absl::Nonnull<const str_format_internal::ParsedFormatBase*> pc) : spec_(pc) {} private: @@ -144,7 +152,7 @@ str_format_internal::StreamedWrapper<T> FormatStreamed(const T& v) { // EXPECT_EQ(8, n); class FormatCountCapture { public: - explicit FormatCountCapture(int* p) : p_(p) {} + explicit FormatCountCapture(absl::Nonnull<int*> p) : p_(p) {} private: // FormatCountCaptureHelper is used to define FormatConvertImpl() for this @@ -153,8 +161,8 @@ class FormatCountCapture { // Unused() is here because of the false positive from -Wunused-private-field // p_ is used in the templated function of the friend FormatCountCaptureHelper // class. - int* Unused() { return p_; } - int* p_; + absl::Nonnull<int*> Unused() { return p_; } + absl::Nonnull<int*> p_; }; // FormatSpec @@ -250,13 +258,13 @@ class FormatCountCapture { // `v` uses `d` for signed integer values, `u` for unsigned integer values, `g` // for floating point values, and formats boolean values as "true"/"false" // (instead of 1 or 0 for booleans formatted using d). `const char*` is not -// supported; please use `std:string` and `string_view`. `char` is also not +// supported; please use `std::string` and `string_view`. `char` is also not // supported due to ambiguity of the type. This specifier does not support // modifiers. // // The `FormatSpec` intrinsically supports all of these fundamental C++ types: // -// * Characters: `char`, `signed char`, `unsigned char` +// * Characters: `char`, `signed char`, `unsigned char`, `wchar_t` // * Integers: `int`, `short`, `unsigned short`, `unsigned`, `long`, // `unsigned long`, `long long`, `unsigned long long` // * Enums: printed as their underlying integral value @@ -264,9 +272,9 @@ class FormatCountCapture { // // However, in the `str_format` library, a format conversion specifies a broader // C++ conceptual category instead of an exact type. For example, `%s` binds to -// any string-like argument, so `std::string`, `absl::string_view`, and -// `const char*` are all accepted. Likewise, `%d` accepts any integer-like -// argument, etc. +// any string-like argument, so `std::string`, `std::wstring`, +// `absl::string_view`, `const char*`, and `const wchar_t*` are all accepted. +// Likewise, `%d` accepts any integer-like argument, etc. template <typename... Args> using FormatSpec = str_format_internal::FormatSpecTemplate< @@ -287,8 +295,8 @@ using FormatSpec = str_format_internal::FormatSpecTemplate< // Example: // // // Verified at compile time. -// absl::ParsedFormat<'s', 'd'> formatString("Welcome to %s, Number %d!"); -// absl::StrFormat(formatString, "TheVillage", 6); +// absl::ParsedFormat<'s', 'd'> format_string("Welcome to %s, Number %d!"); +// absl::StrFormat(format_string, "TheVillage", 6); // // // Verified at runtime. // auto format_runtime = absl::ParsedFormat<'d'>::New(format_string); @@ -369,7 +377,7 @@ ABSL_MUST_USE_RESULT std::string StrFormat(const FormatSpec<Args...>& format, // std::string orig("For example PI is approximately "); // std::cout << StrAppendFormat(&orig, "%12.6f", 3.14); template <typename... Args> -std::string& StrAppendFormat(std::string* dst, +std::string& StrAppendFormat(absl::Nonnull<std::string*> dst, const FormatSpec<Args...>& format, const Args&... args) { return str_format_internal::AppendPack( @@ -381,7 +389,7 @@ std::string& StrAppendFormat(std::string* dst, // // Writes to an output stream given a format string and zero or more arguments, // generally in a manner that is more efficient than streaming the result of -// `absl:: StrFormat()`. The returned object must be streamed before the full +// `absl::StrFormat()`. The returned object must be streamed before the full // expression ends. // // Example: @@ -429,7 +437,7 @@ int PrintF(const FormatSpec<Args...>& format, const Args&... args) { // Outputs: "The capital of Mongolia is Ulaanbaatar" // template <typename... Args> -int FPrintF(std::FILE* output, const FormatSpec<Args...>& format, +int FPrintF(absl::Nonnull<std::FILE*> output, const FormatSpec<Args...>& format, const Args&... args) { return str_format_internal::FprintF( output, str_format_internal::UntypedFormatSpecImpl::Extract(format), @@ -458,8 +466,8 @@ int FPrintF(std::FILE* output, const FormatSpec<Args...>& format, // Post-condition: output == "The capital of Mongolia is Ulaanbaatar" // template <typename... Args> -int SNPrintF(char* output, std::size_t size, const FormatSpec<Args...>& format, - const Args&... args) { +int SNPrintF(absl::Nonnull<char*> output, std::size_t size, + const FormatSpec<Args...>& format, const Args&... args) { return str_format_internal::SnprintF( output, size, str_format_internal::UntypedFormatSpecImpl::Extract(format), {str_format_internal::FormatArgImpl(args)...}); @@ -492,7 +500,7 @@ class FormatRawSink { template <typename T, typename = typename std::enable_if<std::is_constructible< str_format_internal::FormatRawSinkImpl, T*>::value>::type> - FormatRawSink(T* raw) // NOLINT + FormatRawSink(absl::Nonnull<T*> raw) // NOLINT : sink_(raw) {} private: @@ -849,14 +857,16 @@ class FormatSink { } // Support `absl::Format(&sink, format, args...)`. - friend void AbslFormatFlush(FormatSink* sink, absl::string_view v) { + friend void AbslFormatFlush(absl::Nonnull<FormatSink*> sink, + absl::string_view v) { sink->Append(v); } private: friend str_format_internal::FormatSinkImpl; - explicit FormatSink(str_format_internal::FormatSinkImpl* s) : sink_(s) {} - str_format_internal::FormatSinkImpl* sink_; + explicit FormatSink(absl::Nonnull<str_format_internal::FormatSinkImpl*> s) + : sink_(s) {} + absl::Nonnull<str_format_internal::FormatSinkImpl*> sink_; }; // FormatConvertResult diff --git a/contrib/restricted/abseil-cpp/absl/strings/str_join.h b/contrib/restricted/abseil-cpp/absl/strings/str_join.h index ee5ae7efdf..6a92c0f53a 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/str_join.h +++ b/contrib/restricted/abseil-cpp/absl/strings/str_join.h @@ -80,7 +80,7 @@ ABSL_NAMESPACE_BEGIN // absl::StrJoin(v, ", ", [](std::string* out, absl::Duration dur) { // absl::StrAppend(out, absl::FormatDuration(dur)); // }); -// EXPECT_EQ("1s, 10ms", s); +// EXPECT_EQ(s, "1s, 10ms"); // // The following standard formatters are provided within this file: // @@ -164,21 +164,21 @@ DereferenceFormatter() { // // of `absl::string_view` or even `const char*`. // std::vector<std::string> v = {"foo", "bar", "baz"}; // std::string s = absl::StrJoin(v, "-"); -// EXPECT_EQ("foo-bar-baz", s); +// EXPECT_EQ(s, "foo-bar-baz"); // // Example 2: // // Joins the values in the given `std::initializer_list<>` specified using // // brace initialization. This pattern also works with an initializer_list // // of ints or `absl::string_view` -- any `AlphaNum`-compatible type. // std::string s = absl::StrJoin({"foo", "bar", "baz"}, "-"); -// EXPECT_EQ("foo-bar-baz", s); +// EXPECT_EQs, "foo-bar-baz"); // // Example 3: // // Joins a collection of ints. This pattern also works with floats, // // doubles, int64s -- any `StrCat()`-compatible type. // std::vector<int> v = {1, 2, 3, -4}; // std::string s = absl::StrJoin(v, "-"); -// EXPECT_EQ("1-2-3--4", s); +// EXPECT_EQ(s, "1-2-3--4"); // // Example 4: // // Joins a collection of pointer-to-int. By default, pointers are @@ -189,7 +189,7 @@ DereferenceFormatter() { // int x = 1, y = 2, z = 3; // std::vector<int*> v = {&x, &y, &z}; // std::string s = absl::StrJoin(v, "-"); -// EXPECT_EQ("1-2-3", s); +// EXPECT_EQ(s, "1-2-3"); // // Example 5: // // Dereferencing of `std::unique_ptr<>` is also supported: @@ -198,42 +198,42 @@ DereferenceFormatter() { // v.emplace_back(new int(2)); // v.emplace_back(new int(3)); // std::string s = absl::StrJoin(v, "-"); -// EXPECT_EQ("1-2-3", s); +// EXPECT_EQ(s, "1-2-3"); // // Example 6: // // Joins a `std::map`, with each key-value pair separated by an equals // // sign. This pattern would also work with, say, a // // `std::vector<std::pair<>>`. // std::map<std::string, int> m = { -// std::make_pair("a", 1), -// std::make_pair("b", 2), -// std::make_pair("c", 3)}; +// {"a", 1}, +// {"b", 2}, +// {"c", 3}}; // std::string s = absl::StrJoin(m, ",", absl::PairFormatter("=")); -// EXPECT_EQ("a=1,b=2,c=3", s); +// EXPECT_EQ(s, "a=1,b=2,c=3"); // // Example 7: // // These examples show how `absl::StrJoin()` handles a few common edge // // cases: // std::vector<std::string> v_empty; -// EXPECT_EQ("", absl::StrJoin(v_empty, "-")); +// EXPECT_EQ(absl::StrJoin(v_empty, "-"), ""); // // std::vector<std::string> v_one_item = {"foo"}; -// EXPECT_EQ("foo", absl::StrJoin(v_one_item, "-")); +// EXPECT_EQ(absl::StrJoin(v_one_item, "-"), "foo"); // // std::vector<std::string> v_empty_string = {""}; -// EXPECT_EQ("", absl::StrJoin(v_empty_string, "-")); +// EXPECT_EQ(absl::StrJoin(v_empty_string, "-"), ""); // // std::vector<std::string> v_one_item_empty_string = {"a", ""}; -// EXPECT_EQ("a-", absl::StrJoin(v_one_item_empty_string, "-")); +// EXPECT_EQ(absl::StrJoin(v_one_item_empty_string, "-"), "a-"); // // std::vector<std::string> v_two_empty_string = {"", ""}; -// EXPECT_EQ("-", absl::StrJoin(v_two_empty_string, "-")); +// EXPECT_EQ(absl::StrJoin(v_two_empty_string, "-"), "-"); // // Example 8: // // Joins a `std::tuple<T...>` of heterogeneous types, converting each to // // a std::string using the `absl::AlphaNum` class. // std::string s = absl::StrJoin(std::make_tuple(123, "abc", 0.456), "-"); -// EXPECT_EQ("123-abc-0.456", s); +// EXPECT_EQ(s, "123-abc-0.456"); template <typename Iterator, typename Formatter> std::string StrJoin(Iterator start, Iterator end, absl::string_view sep, diff --git a/contrib/restricted/abseil-cpp/absl/strings/str_replace.cc b/contrib/restricted/abseil-cpp/absl/strings/str_replace.cc index 2bd5fa9821..a7ab52fed5 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/str_replace.cc +++ b/contrib/restricted/abseil-cpp/absl/strings/str_replace.cc @@ -14,7 +14,16 @@ #include "absl/strings/str_replace.h" +#include <cstddef> +#include <initializer_list> +#include <string> +#include <utility> +#include <vector> + +#include "absl/base/config.h" +#include "absl/base/nullability.h" #include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -28,8 +37,8 @@ using FixedMapping = // occurred. int ApplySubstitutions( absl::string_view s, - std::vector<strings_internal::ViableSubstitution>* subs_ptr, - std::string* result_ptr) { + absl::Nonnull<std::vector<strings_internal::ViableSubstitution>*> subs_ptr, + absl::Nonnull<std::string*> result_ptr) { auto& subs = *subs_ptr; int substitutions = 0; size_t pos = 0; @@ -74,7 +83,7 @@ std::string StrReplaceAll(absl::string_view s, } int StrReplaceAll(strings_internal::FixedMapping replacements, - std::string* target) { + absl::Nonnull<std::string*> target) { return StrReplaceAll<strings_internal::FixedMapping>(replacements, target); } diff --git a/contrib/restricted/abseil-cpp/absl/strings/str_replace.h b/contrib/restricted/abseil-cpp/absl/strings/str_replace.h index 273c707735..e77ced3e57 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/str_replace.h +++ b/contrib/restricted/abseil-cpp/absl/strings/str_replace.h @@ -43,6 +43,7 @@ #include <vector> #include "absl/base/attributes.h" +#include "absl/base/nullability.h" #include "absl/strings/string_view.h" namespace absl { @@ -113,7 +114,7 @@ std::string StrReplaceAll(absl::string_view s, int StrReplaceAll( std::initializer_list<std::pair<absl::string_view, absl::string_view>> replacements, - std::string* target); + absl::Nonnull<std::string*> target); // Overload of `StrReplaceAll()` to replace patterns within a given output // string *in place* with replacements provided within a container of key/value @@ -128,7 +129,8 @@ int StrReplaceAll( // EXPECT_EQ(count, 2); // EXPECT_EQ("if (ptr < &foo)", s); template <typename StrToStrMapping> -int StrReplaceAll(const StrToStrMapping& replacements, std::string* target); +int StrReplaceAll(const StrToStrMapping& replacements, + absl::Nonnull<std::string*> target); // Implementation details only, past this point. namespace strings_internal { @@ -185,8 +187,8 @@ std::vector<ViableSubstitution> FindSubstitutions( } int ApplySubstitutions(absl::string_view s, - std::vector<ViableSubstitution>* subs_ptr, - std::string* result_ptr); + absl::Nonnull<std::vector<ViableSubstitution>*> subs_ptr, + absl::Nonnull<std::string*> result_ptr); } // namespace strings_internal @@ -201,7 +203,8 @@ std::string StrReplaceAll(absl::string_view s, } template <typename StrToStrMapping> -int StrReplaceAll(const StrToStrMapping& replacements, std::string* target) { +int StrReplaceAll(const StrToStrMapping& replacements, + absl::Nonnull<std::string*> target) { auto subs = strings_internal::FindSubstitutions(*target, replacements); if (subs.empty()) return 0; diff --git a/contrib/restricted/abseil-cpp/absl/strings/str_split.cc b/contrib/restricted/abseil-cpp/absl/strings/str_split.cc index 72ba7c02df..abe486b915 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/str_split.cc +++ b/contrib/restricted/abseil-cpp/absl/strings/str_split.cc @@ -15,16 +15,13 @@ #include "absl/strings/str_split.h" #include <algorithm> -#include <cassert> -#include <cstdint> +#include <cstddef> #include <cstdlib> #include <cstring> -#include <iterator> -#include <limits> -#include <memory> +#include "absl/base/config.h" #include "absl/base/internal/raw_logging.h" -#include "absl/strings/ascii.h" +#include "absl/strings/string_view.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -99,6 +96,11 @@ absl::string_view ByString::Find(absl::string_view text, size_t pos) const { return GenericFind(text, delimiter_, pos, LiteralPolicy()); } +absl::string_view ByAsciiWhitespace::Find(absl::string_view text, + size_t pos) const { + return GenericFind(text, " \t\v\f\r\n", pos, AnyOfPolicy()); +} + // // ByChar // diff --git a/contrib/restricted/abseil-cpp/absl/strings/str_split.h b/contrib/restricted/abseil-cpp/absl/strings/str_split.h index 7bbb68a343..87540278ad 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/str_split.h +++ b/contrib/restricted/abseil-cpp/absl/strings/str_split.h @@ -130,6 +130,24 @@ class ByString { const std::string delimiter_; }; +// ByAsciiWhitespace +// +// A sub-string delimiter that splits by ASCII whitespace +// (space, tab, vertical tab, formfeed, linefeed, or carriage return). +// Note: you probably want to use absl::SkipEmpty() as well! +// +// This class is equivalent to ByAnyChar with ASCII whitespace chars. +// +// Example: +// +// std::vector<std::string> v = absl::StrSplit( +// "a b\tc\n d \n", absl::ByAsciiWhitespace(), absl::SkipEmpty()); +// // v[0] == "a", v[1] == "b", v[2] == "c", v[3] == "d" +class ByAsciiWhitespace { + public: + absl::string_view Find(absl::string_view text, size_t pos) const; +}; + // ByChar // // A single character delimiter. `ByChar` is functionally equivalent to a diff --git a/contrib/restricted/abseil-cpp/absl/strings/string_view.cc b/contrib/restricted/abseil-cpp/absl/strings/string_view.cc index f20ff5306e..97025c32a5 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/string_view.cc +++ b/contrib/restricted/abseil-cpp/absl/strings/string_view.cc @@ -21,6 +21,8 @@ #include <cstring> #include <ostream> +#include "absl/base/nullability.h" + namespace absl { ABSL_NAMESPACE_BEGIN @@ -28,8 +30,10 @@ namespace { // This is significantly faster for case-sensitive matches with very // few possible matches. -const char* memmatch(const char* phaystack, size_t haylen, const char* pneedle, - size_t neelen) { +absl::Nullable<const char*> memmatch(absl::Nullable<const char*> phaystack, + size_t haylen, + absl::Nullable<const char*> pneedle, + size_t neelen) { if (0 == neelen) { return phaystack; // even if haylen is 0 } @@ -37,8 +41,8 @@ const char* memmatch(const char* phaystack, size_t haylen, const char* pneedle, const char* match; const char* hayend = phaystack + haylen - neelen + 1; - // A static cast is used here to work around the fact that memchr returns - // a void* on Posix-compliant systems and const void* on Windows. + // A static cast is used here as memchr returns a const void *, and pointer + // arithmetic is not allowed on pointers to void. while ( (match = static_cast<const char*>(memchr( phaystack, pneedle[0], static_cast<size_t>(hayend - phaystack))))) { @@ -229,7 +233,6 @@ string_view::size_type string_view::find_last_not_of( return npos; } - #ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL constexpr string_view::size_type string_view::npos; constexpr string_view::size_type string_view::kMaxSize; @@ -238,4 +241,22 @@ constexpr string_view::size_type string_view::kMaxSize; ABSL_NAMESPACE_END } // namespace absl +#else + +// https://github.com/abseil/abseil-cpp/issues/1465 +// CMake builds on Apple platforms error when libraries are empty. +// Our CMake configuration can avoid this error on header-only libraries, +// but since this library is conditionally empty, including a single +// variable is an easy workaround. +#ifdef __APPLE__ +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { +extern const char kAvoidEmptyStringViewLibraryWarning; +const char kAvoidEmptyStringViewLibraryWarning = 0; +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl +#endif // __APPLE__ + #endif // ABSL_USES_STD_STRING_VIEW diff --git a/contrib/restricted/abseil-cpp/absl/strings/string_view.h b/contrib/restricted/abseil-cpp/absl/strings/string_view.h index eae11b2ab6..04ca0a38ae 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/string_view.h +++ b/contrib/restricted/abseil-cpp/absl/strings/string_view.h @@ -37,6 +37,7 @@ #include <string> #include "absl/base/attributes.h" +#include "absl/base/nullability.h" #include "absl/base/config.h" #include "absl/base/internal/throw_delegate.h" #include "absl/base/macros.h" @@ -162,11 +163,11 @@ class string_view { public: using traits_type = std::char_traits<char>; using value_type = char; - using pointer = char*; - using const_pointer = const char*; + using pointer = absl::Nullable<char*>; + using const_pointer = absl::Nullable<const char*>; using reference = char&; using const_reference = const char&; - using const_iterator = const char*; + using const_iterator = absl::Nullable<const char*>; using iterator = const_iterator; using const_reverse_iterator = std::reverse_iterator<const_iterator>; using reverse_iterator = const_reverse_iterator; @@ -194,11 +195,12 @@ class string_view { // accepting possibly null strings, use `absl::NullSafeStringView(str)` // instead (see below). // The length check is skipped since it is unnecessary and causes code bloat. - constexpr string_view(const char* str) // NOLINT(runtime/explicit) + constexpr string_view( // NOLINT(runtime/explicit) + absl::Nonnull<const char*> str) : ptr_(str), length_(str ? StrlenInternal(str) : 0) {} // Implicit constructor of a `string_view` from a `const char*` and length. - constexpr string_view(const char* data, size_type len) + constexpr string_view(absl::Nullable<const char*> data, size_type len) : ptr_(data), length_(CheckLengthInternal(len)) {} // NOTE: Harmlessly omitted to work around gdb bug. @@ -427,18 +429,21 @@ class string_view { // Overload of `string_view::compare()` for comparing a `string_view` and a // a different C-style string `s`. - constexpr int compare(const char* s) const { return compare(string_view(s)); } + constexpr int compare(absl::Nonnull<const char*> s) const { + return compare(string_view(s)); + } // Overload of `string_view::compare()` for comparing a substring of the // `string_view` and a different string C-style string `s`. - constexpr int compare(size_type pos1, size_type count1, const char* s) const { + constexpr int compare(size_type pos1, size_type count1, + absl::Nonnull<const char*> s) const { return substr(pos1, count1).compare(string_view(s)); } // Overload of `string_view::compare()` for comparing a substring of the // `string_view` and a substring of a different C-style string `s`. - constexpr int compare(size_type pos1, size_type count1, const char* s, - size_type count2) const { + constexpr int compare(size_type pos1, size_type count1, + absl::Nonnull<const char*> s, size_type count2) const { return substr(pos1, count1).compare(string_view(s, count2)); } @@ -457,13 +462,14 @@ class string_view { // Overload of `string_view::find()` for finding a substring of a different // C-style string `s` within the `string_view`. - size_type find(const char* s, size_type pos, size_type count) const { + size_type find(absl::Nonnull<const char*> s, size_type pos, + size_type count) const { return find(string_view(s, count), pos); } // Overload of `string_view::find()` for finding a different C-style string // `s` within the `string_view`. - size_type find(const char* s, size_type pos = 0) const { + size_type find(absl::Nonnull<const char *> s, size_type pos = 0) const { return find(string_view(s), pos); } @@ -480,13 +486,14 @@ class string_view { // Overload of `string_view::rfind()` for finding a substring of a different // C-style string `s` within the `string_view`. - size_type rfind(const char* s, size_type pos, size_type count) const { + size_type rfind(absl::Nonnull<const char*> s, size_type pos, + size_type count) const { return rfind(string_view(s, count), pos); } // Overload of `string_view::rfind()` for finding a different C-style string // `s` within the `string_view`. - size_type rfind(const char* s, size_type pos = npos) const { + size_type rfind(absl::Nonnull<const char*> s, size_type pos = npos) const { return rfind(string_view(s), pos); } @@ -505,14 +512,15 @@ class string_view { // Overload of `string_view::find_first_of()` for finding a substring of a // different C-style string `s` within the `string_view`. - size_type find_first_of(const char* s, size_type pos, - size_type count) const { + size_type find_first_of(absl::Nonnull<const char*> s, size_type pos, + size_type count) const { return find_first_of(string_view(s, count), pos); } // Overload of `string_view::find_first_of()` for finding a different C-style // string `s` within the `string_view`. - size_type find_first_of(const char* s, size_type pos = 0) const { + size_type find_first_of(absl::Nonnull<const char*> s, + size_type pos = 0) const { return find_first_of(string_view(s), pos); } @@ -531,13 +539,15 @@ class string_view { // Overload of `string_view::find_last_of()` for finding a substring of a // different C-style string `s` within the `string_view`. - size_type find_last_of(const char* s, size_type pos, size_type count) const { + size_type find_last_of(absl::Nonnull<const char*> s, size_type pos, + size_type count) const { return find_last_of(string_view(s, count), pos); } // Overload of `string_view::find_last_of()` for finding a different C-style // string `s` within the `string_view`. - size_type find_last_of(const char* s, size_type pos = npos) const { + size_type find_last_of(absl::Nonnull<const char*> s, + size_type pos = npos) const { return find_last_of(string_view(s), pos); } @@ -554,14 +564,15 @@ class string_view { // Overload of `string_view::find_first_not_of()` for finding a substring of a // different C-style string `s` within the `string_view`. - size_type find_first_not_of(const char* s, size_type pos, + size_type find_first_not_of(absl::Nonnull<const char*> s, size_type pos, size_type count) const { return find_first_not_of(string_view(s, count), pos); } // Overload of `string_view::find_first_not_of()` for finding a different // C-style string `s` within the `string_view`. - size_type find_first_not_of(const char* s, size_type pos = 0) const { + size_type find_first_not_of(absl::Nonnull<const char*> s, + size_type pos = 0) const { return find_first_not_of(string_view(s), pos); } @@ -579,22 +590,76 @@ class string_view { // Overload of `string_view::find_last_not_of()` for finding a substring of a // different C-style string `s` within the `string_view`. - size_type find_last_not_of(const char* s, size_type pos, + size_type find_last_not_of(absl::Nonnull<const char*> s, size_type pos, size_type count) const { return find_last_not_of(string_view(s, count), pos); } // Overload of `string_view::find_last_not_of()` for finding a different // C-style string `s` within the `string_view`. - size_type find_last_not_of(const char* s, size_type pos = npos) const { + size_type find_last_not_of(absl::Nonnull<const char*> s, + size_type pos = npos) const { return find_last_not_of(string_view(s), pos); } +#if ABSL_INTERNAL_CPLUSPLUS_LANG >= 202002L + // string_view::starts_with() + // + // Returns true if the `string_view` starts with the prefix `s`. + // + // This method only exists when targeting at least C++20. + // If support for C++ prior to C++20 is required, use `absl::StartsWith()` + // from `//absl/strings/match.h` for compatibility. + constexpr bool starts_with(string_view s) const noexcept { + return s.empty() || + (size() >= s.size() && + ABSL_INTERNAL_STRING_VIEW_MEMCMP(data(), s.data(), s.size()) == 0); + } + + // Overload of `string_view::starts_with()` that returns true if `c` is the + // first character of the `string_view`. + constexpr bool starts_with(char c) const noexcept { + return !empty() && front() == c; + } + + // Overload of `string_view::starts_with()` that returns true if the + // `string_view` starts with the C-style prefix `s`. + constexpr bool starts_with(const char* s) const { + return starts_with(string_view(s)); + } + + // string_view::ends_with() + // + // Returns true if the `string_view` ends with the suffix `s`. + // + // This method only exists when targeting at least C++20. + // If support for C++ prior to C++20 is required, use `absl::EndsWith()` + // from `//absl/strings/match.h` for compatibility. + constexpr bool ends_with(string_view s) const noexcept { + return s.empty() || (size() >= s.size() && ABSL_INTERNAL_STRING_VIEW_MEMCMP( + data() + (size() - s.size()), + s.data(), s.size()) == 0); + } + + // Overload of `string_view::ends_with()` that returns true if `c` is the + // last character of the `string_view`. + constexpr bool ends_with(char c) const noexcept { + return !empty() && back() == c; + } + + // Overload of `string_view::ends_with()` that returns true if the + // `string_view` ends with the C-style suffix `s`. + constexpr bool ends_with(const char* s) const { + return ends_with(string_view(s)); + } +#endif // ABSL_INTERNAL_CPLUSPLUS_LANG >= 202002L + private: // The constructor from std::string delegates to this constructor. // See the comment on that constructor for the rationale. struct SkipCheckLengthTag {}; - string_view(const char* data, size_type len, SkipCheckLengthTag) noexcept + string_view(absl::Nullable<const char*> data, size_type len, + SkipCheckLengthTag) noexcept : ptr_(data), length_(len) {} static constexpr size_type kMaxSize = @@ -604,7 +669,7 @@ class string_view { return ABSL_HARDENING_ASSERT(len <= kMaxSize), len; } - static constexpr size_type StrlenInternal(const char* str) { + static constexpr size_type StrlenInternal(absl::Nonnull<const char*> str) { #if defined(_MSC_VER) && _MSC_VER >= 1910 && !defined(__clang__) // MSVC 2017+ can evaluate this at compile-time. const char* begin = str; @@ -633,7 +698,7 @@ class string_view { : (compare_result < 0 ? -1 : 1); } - const char* ptr_; + absl::Nullable<const char*> ptr_; size_type length_; }; @@ -694,7 +759,7 @@ inline string_view ClippedSubstr(string_view s, size_t pos, // Creates an `absl::string_view` from a pointer `p` even if it's null-valued. // This function should be used where an `absl::string_view` can be created from // a possibly-null pointer. -constexpr string_view NullSafeStringView(const char* p) { +constexpr string_view NullSafeStringView(absl::Nullable<const char*> p) { return p ? string_view(p) : string_view(); } diff --git a/contrib/restricted/abseil-cpp/absl/strings/strip.h b/contrib/restricted/abseil-cpp/absl/strings/strip.h index 341e66fc92..e3cda5bac3 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/strip.h +++ b/contrib/restricted/abseil-cpp/absl/strings/strip.h @@ -25,6 +25,7 @@ #include <string> #include "absl/base/macros.h" +#include "absl/base/nullability.h" #include "absl/strings/ascii.h" #include "absl/strings/match.h" #include "absl/strings/string_view.h" @@ -43,7 +44,8 @@ ABSL_NAMESPACE_BEGIN // absl::string_view input("abc"); // EXPECT_TRUE(absl::ConsumePrefix(&input, "a")); // EXPECT_EQ(input, "bc"); -inline bool ConsumePrefix(absl::string_view* str, absl::string_view expected) { +inline bool ConsumePrefix(absl::Nonnull<absl::string_view*> str, + absl::string_view expected) { if (!absl::StartsWith(*str, expected)) return false; str->remove_prefix(expected.size()); return true; @@ -59,7 +61,8 @@ inline bool ConsumePrefix(absl::string_view* str, absl::string_view expected) { // absl::string_view input("abcdef"); // EXPECT_TRUE(absl::ConsumeSuffix(&input, "def")); // EXPECT_EQ(input, "abc"); -inline bool ConsumeSuffix(absl::string_view* str, absl::string_view expected) { +inline bool ConsumeSuffix(absl::Nonnull<absl::string_view*> str, + absl::string_view expected) { if (!absl::EndsWith(*str, expected)) return false; str->remove_suffix(expected.size()); return true; diff --git a/contrib/restricted/abseil-cpp/absl/strings/substitute.cc b/contrib/restricted/abseil-cpp/absl/strings/substitute.cc index 33a39305db..dd32c75f69 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/substitute.cc +++ b/contrib/restricted/abseil-cpp/absl/strings/substitute.cc @@ -15,20 +15,28 @@ #include "absl/strings/substitute.h" #include <algorithm> +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <string> +#include "absl/base/config.h" #include "absl/base/internal/raw_logging.h" +#include "absl/base/nullability.h" #include "absl/strings/ascii.h" #include "absl/strings/escaping.h" #include "absl/strings/internal/resize_uninitialized.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" namespace absl { ABSL_NAMESPACE_BEGIN namespace substitute_internal { -void SubstituteAndAppendArray(std::string* output, absl::string_view format, - const absl::string_view* args_array, - size_t num_args) { +void SubstituteAndAppendArray( + absl::Nonnull<std::string*> output, absl::string_view format, + absl::Nullable<const absl::string_view*> args_array, size_t num_args) { // Determine total size needed. size_t size = 0; for (size_t i = 0; i < format.size(); i++) { @@ -97,7 +105,7 @@ void SubstituteAndAppendArray(std::string* output, absl::string_view format, assert(target == output->data() + output->size()); } -Arg::Arg(const void* value) { +Arg::Arg(absl::Nullable<const void*> value) { static_assert(sizeof(scratch_) >= sizeof(value) * 2 + 2, "fix sizeof(scratch_)"); if (value == nullptr) { diff --git a/contrib/restricted/abseil-cpp/absl/strings/substitute.h b/contrib/restricted/abseil-cpp/absl/strings/substitute.h index d6a5a690d7..6c7cba4be5 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/substitute.h +++ b/contrib/restricted/abseil-cpp/absl/strings/substitute.h @@ -78,6 +78,7 @@ #include <vector> #include "absl/base/macros.h" +#include "absl/base/nullability.h" #include "absl/base/port.h" #include "absl/strings/ascii.h" #include "absl/strings/escaping.h" @@ -105,7 +106,7 @@ class Arg { // Overloads for string-y things // // Explicitly overload `const char*` so the compiler doesn't cast to `bool`. - Arg(const char* value) // NOLINT(google-explicit-constructor) + Arg(absl::Nullable<const char*> value) // NOLINT(google-explicit-constructor) : piece_(absl::NullSafeStringView(value)) {} template <typename Allocator> Arg( // NOLINT @@ -176,7 +177,7 @@ class Arg { : piece_(value ? "true" : "false") {} template <typename T, typename = typename std::enable_if< - strings_internal::HasAbslStringify<T>::value>::type> + HasAbslStringify<T>::value>::type> Arg( // NOLINT(google-explicit-constructor) const T& v, strings_internal::StringifySink&& sink = {}) : piece_(strings_internal::ExtractStringification(sink, v)) {} @@ -197,14 +198,15 @@ class Arg { // `void*` values, with the exception of `char*`, are printed as // "0x<hex value>". However, in the case of `nullptr`, "NULL" is printed. - Arg(const void* value); // NOLINT(google-explicit-constructor) + Arg( // NOLINT(google-explicit-constructor) + absl::Nullable<const void*> value); // Normal enums are already handled by the integer formatters. // This overload matches only scoped enums. template <typename T, typename = typename std::enable_if< std::is_enum<T>{} && !std::is_convertible<T, int>{} && - !strings_internal::HasAbslStringify<T>::value>::type> + !HasAbslStringify<T>::value>::type> Arg(T value) // NOLINT(google-explicit-constructor) : Arg(static_cast<typename std::underlying_type<T>::type>(value)) {} @@ -220,12 +222,12 @@ class Arg { // Internal helper function. Don't call this from outside this implementation. // This interface may change without notice. -void SubstituteAndAppendArray(std::string* output, absl::string_view format, - const absl::string_view* args_array, - size_t num_args); +void SubstituteAndAppendArray( + absl::Nonnull<std::string*> output, absl::string_view format, + absl::Nullable<const absl::string_view*> args_array, size_t num_args); #if defined(ABSL_BAD_CALL_IF) -constexpr int CalculateOneBit(const char* format) { +constexpr int CalculateOneBit(absl::Nonnull<const char*> format) { // Returns: // * 2^N for '$N' when N is in [0-9] // * 0 for correct '$' escaping: '$$'. @@ -234,11 +236,11 @@ constexpr int CalculateOneBit(const char* format) { : (1 << (*format - '0')); } -constexpr const char* SkipNumber(const char* format) { +constexpr const char* SkipNumber(absl::Nonnull<const char*> format) { return !*format ? format : (format + 1); } -constexpr int PlaceholderBitmask(const char* format) { +constexpr int PlaceholderBitmask(absl::Nonnull<const char*> format) { return !*format ? 0 : *format != '$' ? PlaceholderBitmask(format + 1) @@ -271,18 +273,21 @@ constexpr int PlaceholderBitmask(const char* format) { // absl::SubstituteAndAppend(boilerplate, format, args...); // } // -inline void SubstituteAndAppend(std::string* output, absl::string_view format) { +inline void SubstituteAndAppend(absl::Nonnull<std::string*> output, + absl::string_view format) { substitute_internal::SubstituteAndAppendArray(output, format, nullptr, 0); } -inline void SubstituteAndAppend(std::string* output, absl::string_view format, +inline void SubstituteAndAppend(absl::Nonnull<std::string*> output, + absl::string_view format, const substitute_internal::Arg& a0) { const absl::string_view args[] = {a0.piece()}; substitute_internal::SubstituteAndAppendArray(output, format, args, ABSL_ARRAYSIZE(args)); } -inline void SubstituteAndAppend(std::string* output, absl::string_view format, +inline void SubstituteAndAppend(absl::Nonnull<std::string*> output, + absl::string_view format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1) { const absl::string_view args[] = {a0.piece(), a1.piece()}; @@ -290,7 +295,8 @@ inline void SubstituteAndAppend(std::string* output, absl::string_view format, ABSL_ARRAYSIZE(args)); } -inline void SubstituteAndAppend(std::string* output, absl::string_view format, +inline void SubstituteAndAppend(absl::Nonnull<std::string*> output, + absl::string_view format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2) { @@ -299,7 +305,8 @@ inline void SubstituteAndAppend(std::string* output, absl::string_view format, ABSL_ARRAYSIZE(args)); } -inline void SubstituteAndAppend(std::string* output, absl::string_view format, +inline void SubstituteAndAppend(absl::Nonnull<std::string*> output, + absl::string_view format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, @@ -310,7 +317,8 @@ inline void SubstituteAndAppend(std::string* output, absl::string_view format, ABSL_ARRAYSIZE(args)); } -inline void SubstituteAndAppend(std::string* output, absl::string_view format, +inline void SubstituteAndAppend(absl::Nonnull<std::string*> output, + absl::string_view format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, @@ -322,27 +330,23 @@ inline void SubstituteAndAppend(std::string* output, absl::string_view format, ABSL_ARRAYSIZE(args)); } -inline void SubstituteAndAppend(std::string* output, absl::string_view format, - const substitute_internal::Arg& a0, - const substitute_internal::Arg& a1, - const substitute_internal::Arg& a2, - const substitute_internal::Arg& a3, - const substitute_internal::Arg& a4, - const substitute_internal::Arg& a5) { +inline void SubstituteAndAppend( + absl::Nonnull<std::string*> output, absl::string_view format, + const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, + const substitute_internal::Arg& a4, const substitute_internal::Arg& a5) { const absl::string_view args[] = {a0.piece(), a1.piece(), a2.piece(), a3.piece(), a4.piece(), a5.piece()}; substitute_internal::SubstituteAndAppendArray(output, format, args, ABSL_ARRAYSIZE(args)); } -inline void SubstituteAndAppend(std::string* output, absl::string_view format, - const substitute_internal::Arg& a0, - const substitute_internal::Arg& a1, - const substitute_internal::Arg& a2, - const substitute_internal::Arg& a3, - const substitute_internal::Arg& a4, - const substitute_internal::Arg& a5, - const substitute_internal::Arg& a6) { +inline void SubstituteAndAppend( + absl::Nonnull<std::string*> output, absl::string_view format, + const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, + const substitute_internal::Arg& a4, const substitute_internal::Arg& a5, + const substitute_internal::Arg& a6) { const absl::string_view args[] = {a0.piece(), a1.piece(), a2.piece(), a3.piece(), a4.piece(), a5.piece(), a6.piece()}; @@ -351,7 +355,7 @@ inline void SubstituteAndAppend(std::string* output, absl::string_view format, } inline void SubstituteAndAppend( - std::string* output, absl::string_view format, + absl::Nonnull<std::string*> output, absl::string_view format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, const substitute_internal::Arg& a5, @@ -364,7 +368,7 @@ inline void SubstituteAndAppend( } inline void SubstituteAndAppend( - std::string* output, absl::string_view format, + absl::Nonnull<std::string*> output, absl::string_view format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, const substitute_internal::Arg& a5, @@ -378,7 +382,7 @@ inline void SubstituteAndAppend( } inline void SubstituteAndAppend( - std::string* output, absl::string_view format, + absl::Nonnull<std::string*> output, absl::string_view format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, const substitute_internal::Arg& a5, @@ -394,14 +398,16 @@ inline void SubstituteAndAppend( #if defined(ABSL_BAD_CALL_IF) // This body of functions catches cases where the number of placeholders // doesn't match the number of data arguments. -void SubstituteAndAppend(std::string* output, const char* format) +void SubstituteAndAppend(absl::Nonnull<std::string*> output, + absl::Nonnull<const char*> format) ABSL_BAD_CALL_IF( substitute_internal::PlaceholderBitmask(format) != 0, "There were no substitution arguments " "but this format string either has a $[0-9] in it or contains " "an unescaped $ character (use $$ instead)"); -void SubstituteAndAppend(std::string* output, const char* format, +void SubstituteAndAppend(absl::Nonnull<std::string*> output, + absl::Nonnull<const char*> format, const substitute_internal::Arg& a0) ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 1, "There was 1 substitution argument given, but " @@ -409,7 +415,8 @@ void SubstituteAndAppend(std::string* output, const char* format, "one of $1-$9, or contains an unescaped $ character (use " "$$ instead)"); -void SubstituteAndAppend(std::string* output, const char* format, +void SubstituteAndAppend(absl::Nonnull<std::string*> output, + absl::Nonnull<const char*> format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1) ABSL_BAD_CALL_IF( @@ -418,7 +425,8 @@ void SubstituteAndAppend(std::string* output, const char* format, "missing its $0/$1, contains one of $2-$9, or contains an " "unescaped $ character (use $$ instead)"); -void SubstituteAndAppend(std::string* output, const char* format, +void SubstituteAndAppend(absl::Nonnull<std::string*> output, + absl::Nonnull<const char*> format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2) @@ -428,7 +436,8 @@ void SubstituteAndAppend(std::string* output, const char* format, "this format string is missing its $0/$1/$2, contains one of " "$3-$9, or contains an unescaped $ character (use $$ instead)"); -void SubstituteAndAppend(std::string* output, const char* format, +void SubstituteAndAppend(absl::Nonnull<std::string*> output, + absl::Nonnull<const char*> format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, @@ -439,7 +448,8 @@ void SubstituteAndAppend(std::string* output, const char* format, "this format string is missing its $0-$3, contains one of " "$4-$9, or contains an unescaped $ character (use $$ instead)"); -void SubstituteAndAppend(std::string* output, const char* format, +void SubstituteAndAppend(absl::Nonnull<std::string*> output, + absl::Nonnull<const char*> format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, @@ -451,13 +461,11 @@ void SubstituteAndAppend(std::string* output, const char* format, "this format string is missing its $0-$4, contains one of " "$5-$9, or contains an unescaped $ character (use $$ instead)"); -void SubstituteAndAppend(std::string* output, const char* format, - const substitute_internal::Arg& a0, - const substitute_internal::Arg& a1, - const substitute_internal::Arg& a2, - const substitute_internal::Arg& a3, - const substitute_internal::Arg& a4, - const substitute_internal::Arg& a5) +void SubstituteAndAppend( + absl::Nonnull<std::string*> output, absl::Nonnull<const char*> format, + const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, + const substitute_internal::Arg& a4, const substitute_internal::Arg& a5) ABSL_BAD_CALL_IF( substitute_internal::PlaceholderBitmask(format) != 63, "There were 6 substitution arguments given, but " @@ -465,10 +473,11 @@ void SubstituteAndAppend(std::string* output, const char* format, "$6-$9, or contains an unescaped $ character (use $$ instead)"); void SubstituteAndAppend( - std::string* output, const char* format, const substitute_internal::Arg& a0, - const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, - const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, - const substitute_internal::Arg& a5, const substitute_internal::Arg& a6) + absl::Nonnull<std::string*> output, absl::Nonnull<const char*> format, + const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, + const substitute_internal::Arg& a4, const substitute_internal::Arg& a5, + const substitute_internal::Arg& a6) ABSL_BAD_CALL_IF( substitute_internal::PlaceholderBitmask(format) != 127, "There were 7 substitution arguments given, but " @@ -476,11 +485,11 @@ void SubstituteAndAppend( "$7-$9, or contains an unescaped $ character (use $$ instead)"); void SubstituteAndAppend( - std::string* output, const char* format, const substitute_internal::Arg& a0, - const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, - const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, - const substitute_internal::Arg& a5, const substitute_internal::Arg& a6, - const substitute_internal::Arg& a7) + absl::Nonnull<std::string*> output, absl::Nonnull<const char*> format, + const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, + const substitute_internal::Arg& a4, const substitute_internal::Arg& a5, + const substitute_internal::Arg& a6, const substitute_internal::Arg& a7) ABSL_BAD_CALL_IF( substitute_internal::PlaceholderBitmask(format) != 255, "There were 8 substitution arguments given, but " @@ -488,11 +497,12 @@ void SubstituteAndAppend( "$8-$9, or contains an unescaped $ character (use $$ instead)"); void SubstituteAndAppend( - std::string* output, const char* format, const substitute_internal::Arg& a0, - const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, - const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, - const substitute_internal::Arg& a5, const substitute_internal::Arg& a6, - const substitute_internal::Arg& a7, const substitute_internal::Arg& a8) + absl::Nonnull<std::string*> output, absl::Nonnull<const char*> format, + const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, + const substitute_internal::Arg& a4, const substitute_internal::Arg& a5, + const substitute_internal::Arg& a6, const substitute_internal::Arg& a7, + const substitute_internal::Arg& a8) ABSL_BAD_CALL_IF( substitute_internal::PlaceholderBitmask(format) != 511, "There were 9 substitution arguments given, but " @@ -500,12 +510,12 @@ void SubstituteAndAppend( "contains an unescaped $ character (use $$ instead)"); void SubstituteAndAppend( - std::string* output, const char* format, const substitute_internal::Arg& a0, - const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, - const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, - const substitute_internal::Arg& a5, const substitute_internal::Arg& a6, - const substitute_internal::Arg& a7, const substitute_internal::Arg& a8, - const substitute_internal::Arg& a9) + absl::Nonnull<std::string*> output, absl::Nonnull<const char*> format, + const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, + const substitute_internal::Arg& a4, const substitute_internal::Arg& a5, + const substitute_internal::Arg& a6, const substitute_internal::Arg& a7, + const substitute_internal::Arg& a8, const substitute_internal::Arg& a9) ABSL_BAD_CALL_IF( substitute_internal::PlaceholderBitmask(format) != 1023, "There were 10 substitution arguments given, but this " @@ -633,20 +643,22 @@ ABSL_MUST_USE_RESULT inline std::string Substitute( #if defined(ABSL_BAD_CALL_IF) // This body of functions catches cases where the number of placeholders // doesn't match the number of data arguments. -std::string Substitute(const char* format) +std::string Substitute(absl::Nonnull<const char*> format) ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 0, "There were no substitution arguments " "but this format string either has a $[0-9] in it or " "contains an unescaped $ character (use $$ instead)"); -std::string Substitute(const char* format, const substitute_internal::Arg& a0) +std::string Substitute(absl::Nonnull<const char*> format, + const substitute_internal::Arg& a0) ABSL_BAD_CALL_IF( substitute_internal::PlaceholderBitmask(format) != 1, "There was 1 substitution argument given, but " "this format string is missing its $0, contains one of $1-$9, " "or contains an unescaped $ character (use $$ instead)"); -std::string Substitute(const char* format, const substitute_internal::Arg& a0, +std::string Substitute(absl::Nonnull<const char*> format, + const substitute_internal::Arg& a0, const substitute_internal::Arg& a1) ABSL_BAD_CALL_IF( substitute_internal::PlaceholderBitmask(format) != 3, @@ -654,7 +666,8 @@ std::string Substitute(const char* format, const substitute_internal::Arg& a0, "this format string is missing its $0/$1, contains one of " "$2-$9, or contains an unescaped $ character (use $$ instead)"); -std::string Substitute(const char* format, const substitute_internal::Arg& a0, +std::string Substitute(absl::Nonnull<const char*> format, + const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2) ABSL_BAD_CALL_IF( @@ -663,7 +676,8 @@ std::string Substitute(const char* format, const substitute_internal::Arg& a0, "this format string is missing its $0/$1/$2, contains one of " "$3-$9, or contains an unescaped $ character (use $$ instead)"); -std::string Substitute(const char* format, const substitute_internal::Arg& a0, +std::string Substitute(absl::Nonnull<const char*> format, + const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, const substitute_internal::Arg& a3) @@ -673,7 +687,8 @@ std::string Substitute(const char* format, const substitute_internal::Arg& a0, "this format string is missing its $0-$3, contains one of " "$4-$9, or contains an unescaped $ character (use $$ instead)"); -std::string Substitute(const char* format, const substitute_internal::Arg& a0, +std::string Substitute(absl::Nonnull<const char*> format, + const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, @@ -684,7 +699,8 @@ std::string Substitute(const char* format, const substitute_internal::Arg& a0, "this format string is missing its $0-$4, contains one of " "$5-$9, or contains an unescaped $ character (use $$ instead)"); -std::string Substitute(const char* format, const substitute_internal::Arg& a0, +std::string Substitute(absl::Nonnull<const char*> format, + const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, @@ -696,27 +712,23 @@ std::string Substitute(const char* format, const substitute_internal::Arg& a0, "this format string is missing its $0-$5, contains one of " "$6-$9, or contains an unescaped $ character (use $$ instead)"); -std::string Substitute(const char* format, const substitute_internal::Arg& a0, - const substitute_internal::Arg& a1, - const substitute_internal::Arg& a2, - const substitute_internal::Arg& a3, - const substitute_internal::Arg& a4, - const substitute_internal::Arg& a5, - const substitute_internal::Arg& a6) +std::string Substitute( + absl::Nonnull<const char*> format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, + const substitute_internal::Arg& a5, const substitute_internal::Arg& a6) ABSL_BAD_CALL_IF( substitute_internal::PlaceholderBitmask(format) != 127, "There were 7 substitution arguments given, but " "this format string is missing its $0-$6, contains one of " "$7-$9, or contains an unescaped $ character (use $$ instead)"); -std::string Substitute(const char* format, const substitute_internal::Arg& a0, - const substitute_internal::Arg& a1, - const substitute_internal::Arg& a2, - const substitute_internal::Arg& a3, - const substitute_internal::Arg& a4, - const substitute_internal::Arg& a5, - const substitute_internal::Arg& a6, - const substitute_internal::Arg& a7) +std::string Substitute( + absl::Nonnull<const char*> format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, + const substitute_internal::Arg& a5, const substitute_internal::Arg& a6, + const substitute_internal::Arg& a7) ABSL_BAD_CALL_IF( substitute_internal::PlaceholderBitmask(format) != 255, "There were 8 substitution arguments given, but " @@ -724,7 +736,7 @@ std::string Substitute(const char* format, const substitute_internal::Arg& a0, "$8-$9, or contains an unescaped $ character (use $$ instead)"); std::string Substitute( - const char* format, const substitute_internal::Arg& a0, + absl::Nonnull<const char*> format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, const substitute_internal::Arg& a5, const substitute_internal::Arg& a6, @@ -736,7 +748,7 @@ std::string Substitute( "contains an unescaped $ character (use $$ instead)"); std::string Substitute( - const char* format, const substitute_internal::Arg& a0, + absl::Nonnull<const char*> format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, const substitute_internal::Arg& a5, const substitute_internal::Arg& a6, diff --git a/contrib/restricted/abseil-cpp/absl/strings/ya.make b/contrib/restricted/abseil-cpp/absl/strings/ya.make index 99b03d7bd6..dffec44d18 100644 --- a/contrib/restricted/abseil-cpp/absl/strings/ya.make +++ b/contrib/restricted/abseil-cpp/absl/strings/ya.make @@ -27,7 +27,7 @@ SRCS( crc/internal/crc.cc crc/internal/crc_cord_state.cc crc/internal/crc_memcpy_fallback.cc - crc/internal/crc_memcpy_x86_64.cc + crc/internal/crc_memcpy_x86_arm_combined.cc crc/internal/crc_non_temporal_memcpy.cc crc/internal/crc_x86_arm_combined.cc status/statusor.cc @@ -45,7 +45,6 @@ SRCS( strings/internal/cord_rep_btree_reader.cc strings/internal/cord_rep_consume.cc strings/internal/cord_rep_crc.cc - strings/internal/cord_rep_ring.cc strings/internal/cordz_functions.cc strings/internal/cordz_handle.cc strings/internal/cordz_info.cc diff --git a/contrib/restricted/abseil-cpp/absl/synchronization/internal/pthread_waiter.h b/contrib/restricted/abseil-cpp/absl/synchronization/internal/pthread_waiter.h index 206aefa47c..23db76ad44 100644 --- a/contrib/restricted/abseil-cpp/absl/synchronization/internal/pthread_waiter.h +++ b/contrib/restricted/abseil-cpp/absl/synchronization/internal/pthread_waiter.h @@ -16,7 +16,7 @@ #ifndef ABSL_SYNCHRONIZATION_INTERNAL_PTHREAD_WAITER_H_ #define ABSL_SYNCHRONIZATION_INTERNAL_PTHREAD_WAITER_H_ -#ifndef _WIN32 +#if !defined(_WIN32) && !defined(__MINGW32__) #include <pthread.h> #include "absl/base/config.h" @@ -55,6 +55,6 @@ class PthreadWaiter : public WaiterCrtp<PthreadWaiter> { ABSL_NAMESPACE_END } // namespace absl -#endif // ndef _WIN32 +#endif // !defined(_WIN32) && !defined(__MINGW32__) #endif // ABSL_SYNCHRONIZATION_INTERNAL_PTHREAD_WAITER_H_ diff --git a/contrib/restricted/abseil-cpp/absl/synchronization/internal/win32_waiter.h b/contrib/restricted/abseil-cpp/absl/synchronization/internal/win32_waiter.h index 87eb617cbd..fdab264e8c 100644 --- a/contrib/restricted/abseil-cpp/absl/synchronization/internal/win32_waiter.h +++ b/contrib/restricted/abseil-cpp/absl/synchronization/internal/win32_waiter.h @@ -20,7 +20,8 @@ #include <sdkddkver.h> #endif -#if defined(_WIN32) && _WIN32_WINNT >= _WIN32_WINNT_VISTA +#if defined(_WIN32) && !defined(__MINGW32__) && \ + _WIN32_WINNT >= _WIN32_WINNT_VISTA #include "absl/base/config.h" #include "absl/synchronization/internal/kernel_timeout.h" @@ -65,6 +66,7 @@ class Win32Waiter : public WaiterCrtp<Win32Waiter> { ABSL_NAMESPACE_END } // namespace absl -#endif // defined(_WIN32) && _WIN32_WINNT >= _WIN32_WINNT_VISTA +#endif // defined(_WIN32) && !defined(__MINGW32__) && + // _WIN32_WINNT >= _WIN32_WINNT_VISTA #endif // ABSL_SYNCHRONIZATION_INTERNAL_WIN32_WAITER_H_ diff --git a/contrib/restricted/abseil-cpp/absl/synchronization/mutex.cc b/contrib/restricted/abseil-cpp/absl/synchronization/mutex.cc index 3aa5560a32..cb3c7e74ec 100644 --- a/contrib/restricted/abseil-cpp/absl/synchronization/mutex.cc +++ b/contrib/restricted/abseil-cpp/absl/synchronization/mutex.cc @@ -129,11 +129,15 @@ enum DelayMode { AGGRESSIVE, GENTLE }; struct ABSL_CACHELINE_ALIGNED MutexGlobals { absl::once_flag once; - int spinloop_iterations = 0; + // Note: this variable is initialized separately in Mutex::LockSlow, + // so that Mutex::Lock does not have a stack frame in optimized build. + std::atomic<int> spinloop_iterations{0}; int32_t mutex_sleep_spins[2] = {}; absl::Duration mutex_sleep_time; }; +ABSL_CONST_INIT static MutexGlobals globals; + absl::Duration MeasureTimeToYield() { absl::Time before = absl::Now(); ABSL_INTERNAL_C_SYMBOL(AbslInternalMutexYield)(); @@ -141,33 +145,30 @@ absl::Duration MeasureTimeToYield() { } const MutexGlobals& GetMutexGlobals() { - ABSL_CONST_INIT static MutexGlobals data; - absl::base_internal::LowLevelCallOnce(&data.once, [&]() { + absl::base_internal::LowLevelCallOnce(&globals.once, [&]() { if (absl::base_internal::NumCPUs() > 1) { - // If this is multiprocessor, allow spinning. If the mode is - // aggressive then spin many times before yielding. If the mode is - // gentle then spin only a few times before yielding. Aggressive spinning - // is used to ensure that an Unlock() call, which must get the spin lock - // for any thread to make progress gets it without undue delay. - data.spinloop_iterations = 1500; - data.mutex_sleep_spins[AGGRESSIVE] = 5000; - data.mutex_sleep_spins[GENTLE] = 250; - data.mutex_sleep_time = absl::Microseconds(10); + // If the mode is aggressive then spin many times before yielding. + // If the mode is gentle then spin only a few times before yielding. + // Aggressive spinning is used to ensure that an Unlock() call, + // which must get the spin lock for any thread to make progress gets it + // without undue delay. + globals.mutex_sleep_spins[AGGRESSIVE] = 5000; + globals.mutex_sleep_spins[GENTLE] = 250; + globals.mutex_sleep_time = absl::Microseconds(10); } else { // If this a uniprocessor, only yield/sleep. Real-time threads are often // unable to yield, so the sleep time needs to be long enough to keep // the calling thread asleep until scheduling happens. - data.spinloop_iterations = 0; - data.mutex_sleep_spins[AGGRESSIVE] = 0; - data.mutex_sleep_spins[GENTLE] = 0; - data.mutex_sleep_time = MeasureTimeToYield() * 5; - data.mutex_sleep_time = - std::min(data.mutex_sleep_time, absl::Milliseconds(1)); - data.mutex_sleep_time = - std::max(data.mutex_sleep_time, absl::Microseconds(10)); + globals.mutex_sleep_spins[AGGRESSIVE] = 0; + globals.mutex_sleep_spins[GENTLE] = 0; + globals.mutex_sleep_time = MeasureTimeToYield() * 5; + globals.mutex_sleep_time = + std::min(globals.mutex_sleep_time, absl::Milliseconds(1)); + globals.mutex_sleep_time = + std::max(globals.mutex_sleep_time, absl::Microseconds(10)); } }); - return data; + return globals; } } // namespace @@ -202,31 +203,23 @@ int MutexDelay(int32_t c, int mode) { // Ensure that "(*pv & bits) == bits" by doing an atomic update of "*pv" to // "*pv | bits" if necessary. Wait until (*pv & wait_until_clear)==0 // before making any change. +// Returns true if bits were previously unset and set by the call. // This is used to set flags in mutex and condition variable words. -static void AtomicSetBits(std::atomic<intptr_t>* pv, intptr_t bits, +static bool AtomicSetBits(std::atomic<intptr_t>* pv, intptr_t bits, intptr_t wait_until_clear) { - intptr_t v; - do { - v = pv->load(std::memory_order_relaxed); - } while ((v & bits) != bits && - ((v & wait_until_clear) != 0 || - !pv->compare_exchange_weak(v, v | bits, std::memory_order_release, - std::memory_order_relaxed))); -} - -// Ensure that "(*pv & bits) == 0" by doing an atomic update of "*pv" to -// "*pv & ~bits" if necessary. Wait until (*pv & wait_until_clear)==0 -// before making any change. -// This is used to unset flags in mutex and condition variable words. -static void AtomicClearBits(std::atomic<intptr_t>* pv, intptr_t bits, - intptr_t wait_until_clear) { - intptr_t v; - do { - v = pv->load(std::memory_order_relaxed); - } while ((v & bits) != 0 && - ((v & wait_until_clear) != 0 || - !pv->compare_exchange_weak(v, v & ~bits, std::memory_order_release, - std::memory_order_relaxed))); + for (;;) { + intptr_t v = pv->load(std::memory_order_relaxed); + if ((v & bits) == bits) { + return false; + } + if ((v & wait_until_clear) != 0) { + continue; + } + if (pv->compare_exchange_weak(v, v | bits, std::memory_order_release, + std::memory_order_relaxed)) { + return true; + } + } } //------------------------------------------------------------------ @@ -337,12 +330,49 @@ static SynchEvent* EnsureSynchEvent(std::atomic<intptr_t>* addr, const char* name, intptr_t bits, intptr_t lockbit) { uint32_t h = reinterpret_cast<uintptr_t>(addr) % kNSynchEvent; - SynchEvent* e; - // first look for existing SynchEvent struct.. synch_event_mu.Lock(); - for (e = synch_event[h]; - e != nullptr && e->masked_addr != base_internal::HidePtr(addr); - e = e->next) { + // When a Mutex/CondVar is destroyed, we don't remove the associated + // SynchEvent to keep destructors empty in release builds for performance + // reasons. If the current call is the first to set bits (kMuEvent/kCVEvent), + // we don't look up the existing even because (if it exists, it must be for + // the previous Mutex/CondVar that existed at the same address). + // The leaking events must not be a problem for tests, which should create + // bounded amount of events. And debug logging is not supposed to be enabled + // in production. However, if it's accidentally enabled, or briefly enabled + // for some debugging, we don't want to crash the program. Instead we drop + // all events, if we accumulated too many of them. Size of a single event + // is ~48 bytes, so 100K events is ~5 MB. + // Additionally we could delete the old event for the same address, + // but it would require a better hashmap (if we accumulate too many events, + // linked lists will grow and traversing them will be very slow). + constexpr size_t kMaxSynchEventCount = 100 << 10; + // Total number of live synch events. + static size_t synch_event_count ABSL_GUARDED_BY(synch_event_mu); + if (++synch_event_count > kMaxSynchEventCount) { + synch_event_count = 0; + ABSL_RAW_LOG(ERROR, + "Accumulated %zu Mutex debug objects. If you see this" + " in production, it may mean that the production code" + " accidentally calls " + "Mutex/CondVar::EnableDebugLog/EnableInvariantDebugging.", + kMaxSynchEventCount); + for (auto*& head : synch_event) { + for (auto* e = head; e != nullptr;) { + SynchEvent* next = e->next; + if (--(e->refcount) == 0) { + base_internal::LowLevelAlloc::Free(e); + } + e = next; + } + head = nullptr; + } + } + SynchEvent* e = nullptr; + if (!AtomicSetBits(addr, bits, lockbit)) { + for (e = synch_event[h]; + e != nullptr && e->masked_addr != base_internal::HidePtr(addr); + e = e->next) { + } } if (e == nullptr) { // no SynchEvent struct found; make one. if (name == nullptr) { @@ -358,7 +388,6 @@ static SynchEvent* EnsureSynchEvent(std::atomic<intptr_t>* addr, e->log = false; strcpy(e->name, name); // NOLINT(runtime/printf) e->next = synch_event[h]; - AtomicSetBits(addr, bits, lockbit); synch_event[h] = e; } else { e->refcount++; // for return value @@ -367,11 +396,6 @@ static SynchEvent* EnsureSynchEvent(std::atomic<intptr_t>* addr, return e; } -// Deallocate the SynchEvent *e, whose refcount has fallen to zero. -static void DeleteSynchEvent(SynchEvent* e) { - base_internal::LowLevelAlloc::Free(e); -} - // Decrement the reference count of *e, or do nothing if e==null. static void UnrefSynchEvent(SynchEvent* e) { if (e != nullptr) { @@ -379,36 +403,11 @@ static void UnrefSynchEvent(SynchEvent* e) { bool del = (--(e->refcount) == 0); synch_event_mu.Unlock(); if (del) { - DeleteSynchEvent(e); + base_internal::LowLevelAlloc::Free(e); } } } -// Forget the mapping from the object (Mutex or CondVar) at address addr -// to SynchEvent object, and clear "bits" in its word (waiting until lockbit -// is clear before doing so). -static void ForgetSynchEvent(std::atomic<intptr_t>* addr, intptr_t bits, - intptr_t lockbit) { - uint32_t h = reinterpret_cast<uintptr_t>(addr) % kNSynchEvent; - SynchEvent** pe; - SynchEvent* e; - synch_event_mu.Lock(); - for (pe = &synch_event[h]; - (e = *pe) != nullptr && e->masked_addr != base_internal::HidePtr(addr); - pe = &e->next) { - } - bool del = false; - if (e != nullptr) { - *pe = e->next; - del = (--(e->refcount) == 0); - } - AtomicClearBits(addr, bits, lockbit); - synch_event_mu.Unlock(); - if (del) { - DeleteSynchEvent(e); - } -} - // Return a refcounted reference to the SynchEvent of the object at address // "addr", if any. The pointer returned is valid until the UnrefSynchEvent() is // called. @@ -579,31 +578,25 @@ static SynchLocksHeld* Synch_GetAllLocks() { // Post on "w"'s associated PerThreadSem. void Mutex::IncrementSynchSem(Mutex* mu, PerThreadSynch* w) { - if (mu) { - ABSL_TSAN_MUTEX_PRE_DIVERT(mu, 0); - // We miss synchronization around passing PerThreadSynch between threads - // since it happens inside of the Mutex code, so we need to ignore all - // accesses to the object. - ABSL_ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN(); - PerThreadSem::Post(w->thread_identity()); - ABSL_ANNOTATE_IGNORE_READS_AND_WRITES_END(); - ABSL_TSAN_MUTEX_POST_DIVERT(mu, 0); - } else { - PerThreadSem::Post(w->thread_identity()); - } + static_cast<void>(mu); // Prevent unused param warning in non-TSAN builds. + ABSL_TSAN_MUTEX_PRE_DIVERT(mu, 0); + // We miss synchronization around passing PerThreadSynch between threads + // since it happens inside of the Mutex code, so we need to ignore all + // accesses to the object. + ABSL_ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN(); + PerThreadSem::Post(w->thread_identity()); + ABSL_ANNOTATE_IGNORE_READS_AND_WRITES_END(); + ABSL_TSAN_MUTEX_POST_DIVERT(mu, 0); } // Wait on "w"'s associated PerThreadSem; returns false if timeout expired. bool Mutex::DecrementSynchSem(Mutex* mu, PerThreadSynch* w, KernelTimeout t) { - if (mu) { - ABSL_TSAN_MUTEX_PRE_DIVERT(mu, 0); - } + static_cast<void>(mu); // Prevent unused param warning in non-TSAN builds. + ABSL_TSAN_MUTEX_PRE_DIVERT(mu, 0); assert(w == Synch_GetPerThread()); static_cast<void>(w); bool res = PerThreadSem::Wait(t); - if (mu) { - ABSL_TSAN_MUTEX_POST_DIVERT(mu, 0); - } + ABSL_TSAN_MUTEX_POST_DIVERT(mu, 0); return res; } @@ -682,6 +675,7 @@ static const intptr_t kMuOne = 0x0100; // a count of one reader // flags passed to Enqueue and LockSlow{,WithTimeout,Loop} static const int kMuHasBlocked = 0x01; // already blocked (MUST == 1) static const int kMuIsCond = 0x02; // conditional waiter (CV or Condition) +static const int kMuIsFer = 0x04; // wait morphing from a CondVar static_assert(PerThreadSynch::kAlignment > kMuLow, "PerThreadSynch::kAlignment must be greater than kMuLow"); @@ -737,25 +731,43 @@ static unsigned TsanFlags(Mutex::MuHow how) { } #endif -static bool DebugOnlyIsExiting() { - return false; -} +#if defined(__APPLE__) || defined(ABSL_BUILD_DLL) +// When building a dll symbol export lists may reference the destructor +// and want it to be an exported symbol rather than an inline function. +// Some apple builds also do dynamic library build but don't say it explicitly. +Mutex::~Mutex() { Dtor(); } +#endif -Mutex::~Mutex() { - intptr_t v = mu_.load(std::memory_order_relaxed); - if ((v & kMuEvent) != 0 && !DebugOnlyIsExiting()) { - ForgetSynchEvent(&this->mu_, kMuEvent, kMuSpin); - } +#if !defined(NDEBUG) || defined(ABSL_HAVE_THREAD_SANITIZER) +void Mutex::Dtor() { if (kDebugMode) { this->ForgetDeadlockInfo(); } ABSL_TSAN_MUTEX_DESTROY(this, __tsan_mutex_not_static); } +#endif void Mutex::EnableDebugLog(const char* name) { + // Need to disable writes here and in EnableInvariantDebugging to prevent + // false race reports on SynchEvent objects. TSan ignores synchronization + // on synch_event_mu in Lock/Unlock/etc methods due to mutex annotations, + // but it sees few accesses to SynchEvent in EvalConditionAnnotated. + // If we don't ignore accesses here, it can result in false races + // between EvalConditionAnnotated and SynchEvent reuse in EnsureSynchEvent. + ABSL_ANNOTATE_IGNORE_WRITES_BEGIN(); SynchEvent* e = EnsureSynchEvent(&this->mu_, name, kMuEvent, kMuSpin); e->log = true; UnrefSynchEvent(e); + // This prevents "error: undefined symbol: absl::Mutex::~Mutex()" + // in a release build (NDEBUG defined) when a test does "#undef NDEBUG" + // to use assert macro. In such case, the test does not get the dtor + // definition because it's supposed to be outline when NDEBUG is not defined, + // and this source file does not define one either because NDEBUG is defined. + // Since it's not possible to take address of a destructor, we move the + // actual destructor code into the separate Dtor function and force the + // compiler to emit this function even if it's inline by taking its address. + ABSL_ATTRIBUTE_UNUSED volatile auto dtor = &Mutex::Dtor; + ABSL_ANNOTATE_IGNORE_WRITES_END(); } void EnableMutexInvariantDebugging(bool enabled) { @@ -763,6 +775,7 @@ void EnableMutexInvariantDebugging(bool enabled) { } void Mutex::EnableInvariantDebugging(void (*invariant)(void*), void* arg) { + ABSL_ANNOTATE_IGNORE_WRITES_BEGIN(); if (synch_check_invariants.load(std::memory_order_acquire) && invariant != nullptr) { SynchEvent* e = EnsureSynchEvent(&this->mu_, nullptr, kMuEvent, kMuSpin); @@ -770,6 +783,7 @@ void Mutex::EnableInvariantDebugging(void (*invariant)(void*), void* arg) { e->arg = arg; UnrefSynchEvent(e); } + ABSL_ANNOTATE_IGNORE_WRITES_END(); } void SetMutexDeadlockDetectionMode(OnDeadlockCycle mode) { @@ -920,20 +934,23 @@ static PerThreadSynch* Enqueue(PerThreadSynch* head, SynchWaitParams* waitp, s->wake = false; // not being woken s->cond_waiter = ((flags & kMuIsCond) != 0); #ifdef ABSL_HAVE_PTHREAD_GETSCHEDPARAM - int64_t now_cycles = CycleClock::Now(); - if (s->next_priority_read_cycles < now_cycles) { - // Every so often, update our idea of the thread's priority. - // pthread_getschedparam() is 5% of the block/wakeup time; - // CycleClock::Now() is 0.5%. - int policy; - struct sched_param param; - const int err = pthread_getschedparam(pthread_self(), &policy, ¶m); - if (err != 0) { - ABSL_RAW_LOG(ERROR, "pthread_getschedparam failed: %d", err); - } else { - s->priority = param.sched_priority; - s->next_priority_read_cycles = - now_cycles + static_cast<int64_t>(CycleClock::Frequency()); + if ((flags & kMuIsFer) == 0) { + assert(s == Synch_GetPerThread()); + int64_t now_cycles = CycleClock::Now(); + if (s->next_priority_read_cycles < now_cycles) { + // Every so often, update our idea of the thread's priority. + // pthread_getschedparam() is 5% of the block/wakeup time; + // CycleClock::Now() is 0.5%. + int policy; + struct sched_param param; + const int err = pthread_getschedparam(pthread_self(), &policy, ¶m); + if (err != 0) { + ABSL_RAW_LOG(ERROR, "pthread_getschedparam failed: %d", err); + } else { + s->priority = param.sched_priority; + s->next_priority_read_cycles = + now_cycles + static_cast<int64_t>(CycleClock::Frequency()); + } } } #endif @@ -962,8 +979,7 @@ static PerThreadSynch* Enqueue(PerThreadSynch* head, SynchWaitParams* waitp, } while (s->priority <= advance_to->priority); // termination guaranteed because s->priority > head->priority // and head is the end of a skip chain - } else if (waitp->how == kExclusive && - Condition::GuaranteedEqual(waitp->cond, nullptr)) { + } else if (waitp->how == kExclusive && waitp->cond == nullptr) { // An unlocker could be scanning the queue, but we know it will recheck // the queue front for writers that have no condition, which is what s // is, so an insert at front is safe. @@ -994,6 +1010,24 @@ static PerThreadSynch* Enqueue(PerThreadSynch* head, SynchWaitParams* waitp, if (MuEquivalentWaiter(s, s->next)) { // s->may_skip is known to be true s->skip = s->next; // s may skip to its successor } + } else if ((flags & kMuHasBlocked) && + (s->priority >= head->next->priority) && + (!head->maybe_unlocking || + (waitp->how == kExclusive && + Condition::GuaranteedEqual(waitp->cond, nullptr)))) { + // This thread has already waited, then was woken, then failed to acquire + // the mutex and now tries to requeue. Try to requeue it at head, + // otherwise it can suffer bad latency (wait whole queue several times). + // However, we need to be conservative. First, we need to ensure that we + // respect priorities. Then, we need to be careful to not break wait + // queue invariants: we require either that unlocker is not scanning + // the queue or that the current thread is a writer with no condition + // (unlocker will recheck the queue for such waiters). + s->next = head->next; + head->next = s; + if (MuEquivalentWaiter(s, s->next)) { // s->may_skip is known to be true + s->skip = s->next; // s may skip to its successor + } } else { // enqueue not done any other way, so // we're inserting s at the back // s will become new head; copy data from head into it @@ -1472,7 +1506,7 @@ void Mutex::AssertNotHeld() const { // Attempt to acquire *mu, and return whether successful. The implementation // may spin for a short while if the lock cannot be acquired immediately. static bool TryAcquireWithSpinning(std::atomic<intptr_t>* mu) { - int c = GetMutexGlobals().spinloop_iterations; + int c = globals.spinloop_iterations.load(std::memory_order_relaxed); do { // do/while somewhat faster on AMD intptr_t v = mu->load(std::memory_order_relaxed); if ((v & (kMuReader | kMuEvent)) != 0) { @@ -1492,11 +1526,12 @@ void Mutex::Lock() { GraphId id = DebugOnlyDeadlockCheck(this); intptr_t v = mu_.load(std::memory_order_relaxed); // try fast acquire, then spin loop - if ((v & (kMuWriter | kMuReader | kMuEvent)) != 0 || - !mu_.compare_exchange_strong(v, kMuWriter | v, std::memory_order_acquire, - std::memory_order_relaxed)) { + if (ABSL_PREDICT_FALSE((v & (kMuWriter | kMuReader | kMuEvent)) != 0) || + ABSL_PREDICT_FALSE(!mu_.compare_exchange_strong( + v, kMuWriter | v, std::memory_order_acquire, + std::memory_order_relaxed))) { // try spin acquire, then slow loop - if (!TryAcquireWithSpinning(&this->mu_)) { + if (ABSL_PREDICT_FALSE(!TryAcquireWithSpinning(&this->mu_))) { this->LockSlow(kExclusive, nullptr, 0); } } @@ -1508,159 +1543,95 @@ void Mutex::ReaderLock() { ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_read_lock); GraphId id = DebugOnlyDeadlockCheck(this); intptr_t v = mu_.load(std::memory_order_relaxed); - // try fast acquire, then slow loop - if ((v & (kMuWriter | kMuWait | kMuEvent)) != 0 || - !mu_.compare_exchange_strong(v, (kMuReader | v) + kMuOne, - std::memory_order_acquire, - std::memory_order_relaxed)) { - this->LockSlow(kShared, nullptr, 0); + for (;;) { + // If there are non-readers holding the lock, use the slow loop. + if (ABSL_PREDICT_FALSE(v & (kMuWriter | kMuWait | kMuEvent)) != 0) { + this->LockSlow(kShared, nullptr, 0); + break; + } + // We can avoid the loop and only use the CAS when the lock is free or + // only held by readers. + if (ABSL_PREDICT_TRUE(mu_.compare_exchange_weak( + v, (kMuReader | v) + kMuOne, std::memory_order_acquire, + std::memory_order_relaxed))) { + break; + } } DebugOnlyLockEnter(this, id); ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_read_lock, 0); } -void Mutex::LockWhen(const Condition& cond) { - ABSL_TSAN_MUTEX_PRE_LOCK(this, 0); - GraphId id = DebugOnlyDeadlockCheck(this); - this->LockSlow(kExclusive, &cond, 0); - DebugOnlyLockEnter(this, id); - ABSL_TSAN_MUTEX_POST_LOCK(this, 0, 0); -} - -bool Mutex::LockWhenWithTimeout(const Condition& cond, absl::Duration timeout) { - ABSL_TSAN_MUTEX_PRE_LOCK(this, 0); - GraphId id = DebugOnlyDeadlockCheck(this); - bool res = LockSlowWithDeadline(kExclusive, &cond, KernelTimeout(timeout), 0); - DebugOnlyLockEnter(this, id); - ABSL_TSAN_MUTEX_POST_LOCK(this, 0, 0); - return res; -} - -bool Mutex::LockWhenWithDeadline(const Condition& cond, absl::Time deadline) { - ABSL_TSAN_MUTEX_PRE_LOCK(this, 0); - GraphId id = DebugOnlyDeadlockCheck(this); - bool res = - LockSlowWithDeadline(kExclusive, &cond, KernelTimeout(deadline), 0); - DebugOnlyLockEnter(this, id); - ABSL_TSAN_MUTEX_POST_LOCK(this, 0, 0); - return res; -} - -void Mutex::ReaderLockWhen(const Condition& cond) { - ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_read_lock); - GraphId id = DebugOnlyDeadlockCheck(this); - this->LockSlow(kShared, &cond, 0); - DebugOnlyLockEnter(this, id); - ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_read_lock, 0); -} - -bool Mutex::ReaderLockWhenWithTimeout(const Condition& cond, - absl::Duration timeout) { - ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_read_lock); - GraphId id = DebugOnlyDeadlockCheck(this); - bool res = LockSlowWithDeadline(kShared, &cond, KernelTimeout(timeout), 0); - DebugOnlyLockEnter(this, id); - ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_read_lock, 0); - return res; -} - -bool Mutex::ReaderLockWhenWithDeadline(const Condition& cond, - absl::Time deadline) { - ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_read_lock); +bool Mutex::LockWhenCommon(const Condition& cond, + synchronization_internal::KernelTimeout t, + bool write) { + MuHow how = write ? kExclusive : kShared; + ABSL_TSAN_MUTEX_PRE_LOCK(this, TsanFlags(how)); GraphId id = DebugOnlyDeadlockCheck(this); - bool res = LockSlowWithDeadline(kShared, &cond, KernelTimeout(deadline), 0); + bool res = LockSlowWithDeadline(how, &cond, t, 0); DebugOnlyLockEnter(this, id); - ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_read_lock, 0); + ABSL_TSAN_MUTEX_POST_LOCK(this, TsanFlags(how), 0); return res; } -void Mutex::Await(const Condition& cond) { - if (cond.Eval()) { // condition already true; nothing to do - if (kDebugMode) { - this->AssertReaderHeld(); - } - } else { // normal case - ABSL_RAW_CHECK(this->AwaitCommon(cond, KernelTimeout::Never()), - "condition untrue on return from Await"); - } -} - -bool Mutex::AwaitWithTimeout(const Condition& cond, absl::Duration timeout) { - if (cond.Eval()) { // condition already true; nothing to do - if (kDebugMode) { - this->AssertReaderHeld(); - } - return true; +bool Mutex::AwaitCommon(const Condition& cond, KernelTimeout t) { + if (kDebugMode) { + this->AssertReaderHeld(); } - - KernelTimeout t{timeout}; - bool res = this->AwaitCommon(cond, t); - ABSL_RAW_CHECK(res || t.has_timeout(), - "condition untrue on return from Await"); - return res; -} - -bool Mutex::AwaitWithDeadline(const Condition& cond, absl::Time deadline) { if (cond.Eval()) { // condition already true; nothing to do - if (kDebugMode) { - this->AssertReaderHeld(); - } return true; } - - KernelTimeout t{deadline}; - bool res = this->AwaitCommon(cond, t); - ABSL_RAW_CHECK(res || t.has_timeout(), - "condition untrue on return from Await"); - return res; -} - -bool Mutex::AwaitCommon(const Condition& cond, KernelTimeout t) { - this->AssertReaderHeld(); MuHow how = (mu_.load(std::memory_order_relaxed) & kMuWriter) ? kExclusive : kShared; ABSL_TSAN_MUTEX_PRE_UNLOCK(this, TsanFlags(how)); SynchWaitParams waitp(how, &cond, t, nullptr /*no cvmu*/, Synch_GetPerThreadAnnotated(this), nullptr /*no cv_word*/); - int flags = kMuHasBlocked; - if (!Condition::GuaranteedEqual(&cond, nullptr)) { - flags |= kMuIsCond; - } this->UnlockSlow(&waitp); this->Block(waitp.thread); ABSL_TSAN_MUTEX_POST_UNLOCK(this, TsanFlags(how)); ABSL_TSAN_MUTEX_PRE_LOCK(this, TsanFlags(how)); - this->LockSlowLoop(&waitp, flags); + this->LockSlowLoop(&waitp, kMuHasBlocked | kMuIsCond); bool res = waitp.cond != nullptr || // => cond known true from LockSlowLoop EvalConditionAnnotated(&cond, this, true, false, how == kShared); ABSL_TSAN_MUTEX_POST_LOCK(this, TsanFlags(how), 0); + ABSL_RAW_CHECK(res || t.has_timeout(), + "condition untrue on return from Await"); return res; } bool Mutex::TryLock() { ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_try_lock); intptr_t v = mu_.load(std::memory_order_relaxed); - if ((v & (kMuWriter | kMuReader | kMuEvent)) == 0 && // try fast acquire - mu_.compare_exchange_strong(v, kMuWriter | v, std::memory_order_acquire, - std::memory_order_relaxed)) { - DebugOnlyLockEnter(this); - ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_try_lock, 0); - return true; - } - if ((v & kMuEvent) != 0) { // we're recording events - if ((v & kExclusive->slow_need_zero) == 0 && // try fast acquire - mu_.compare_exchange_strong( - v, (kExclusive->fast_or | v) + kExclusive->fast_add, - std::memory_order_acquire, std::memory_order_relaxed)) { + // Try fast acquire. + if (ABSL_PREDICT_TRUE((v & (kMuWriter | kMuReader | kMuEvent)) == 0)) { + if (ABSL_PREDICT_TRUE(mu_.compare_exchange_strong( + v, kMuWriter | v, std::memory_order_acquire, + std::memory_order_relaxed))) { DebugOnlyLockEnter(this); - PostSynchEvent(this, SYNCH_EV_TRYLOCK_SUCCESS); ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_try_lock, 0); return true; - } else { - PostSynchEvent(this, SYNCH_EV_TRYLOCK_FAILED); } + } else if (ABSL_PREDICT_FALSE((v & kMuEvent) != 0)) { + // We're recording events. + return TryLockSlow(); + } + ABSL_TSAN_MUTEX_POST_LOCK( + this, __tsan_mutex_try_lock | __tsan_mutex_try_lock_failed, 0); + return false; +} + +ABSL_ATTRIBUTE_NOINLINE bool Mutex::TryLockSlow() { + intptr_t v = mu_.load(std::memory_order_relaxed); + if ((v & kExclusive->slow_need_zero) == 0 && // try fast acquire + mu_.compare_exchange_strong( + v, (kExclusive->fast_or | v) + kExclusive->fast_add, + std::memory_order_acquire, std::memory_order_relaxed)) { + DebugOnlyLockEnter(this); + PostSynchEvent(this, SYNCH_EV_TRYLOCK_SUCCESS); + ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_try_lock, 0); + return true; } + PostSynchEvent(this, SYNCH_EV_TRYLOCK_FAILED); ABSL_TSAN_MUTEX_POST_LOCK( this, __tsan_mutex_try_lock | __tsan_mutex_try_lock_failed, 0); return false; @@ -1670,41 +1641,57 @@ bool Mutex::ReaderTryLock() { ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_read_lock | __tsan_mutex_try_lock); intptr_t v = mu_.load(std::memory_order_relaxed); + // Clang tends to unroll the loop when compiling with optimization. + // But in this case it just unnecessary increases code size. + // If CAS is failing due to contention, the jump cost is negligible. +#if defined(__clang__) +#pragma nounroll +#endif // The while-loops (here and below) iterate only if the mutex word keeps - // changing (typically because the reader count changes) under the CAS. We - // limit the number of attempts to avoid having to think about livelock. - int loop_limit = 5; - while ((v & (kMuWriter | kMuWait | kMuEvent)) == 0 && loop_limit != 0) { - if (mu_.compare_exchange_strong(v, (kMuReader | v) + kMuOne, - std::memory_order_acquire, - std::memory_order_relaxed)) { + // changing (typically because the reader count changes) under the CAS. + // We limit the number of attempts to avoid having to think about livelock. + for (int loop_limit = 5; loop_limit != 0; loop_limit--) { + if (ABSL_PREDICT_FALSE((v & (kMuWriter | kMuWait | kMuEvent)) != 0)) { + break; + } + if (ABSL_PREDICT_TRUE(mu_.compare_exchange_strong( + v, (kMuReader | v) + kMuOne, std::memory_order_acquire, + std::memory_order_relaxed))) { DebugOnlyLockEnter(this); ABSL_TSAN_MUTEX_POST_LOCK( this, __tsan_mutex_read_lock | __tsan_mutex_try_lock, 0); return true; } - loop_limit--; - v = mu_.load(std::memory_order_relaxed); } - if ((v & kMuEvent) != 0) { // we're recording events - loop_limit = 5; - while ((v & kShared->slow_need_zero) == 0 && loop_limit != 0) { - if (mu_.compare_exchange_strong(v, (kMuReader | v) + kMuOne, - std::memory_order_acquire, - std::memory_order_relaxed)) { - DebugOnlyLockEnter(this); - PostSynchEvent(this, SYNCH_EV_READERTRYLOCK_SUCCESS); - ABSL_TSAN_MUTEX_POST_LOCK( - this, __tsan_mutex_read_lock | __tsan_mutex_try_lock, 0); - return true; - } - loop_limit--; - v = mu_.load(std::memory_order_relaxed); - } - if ((v & kMuEvent) != 0) { - PostSynchEvent(this, SYNCH_EV_READERTRYLOCK_FAILED); + if (ABSL_PREDICT_TRUE((v & kMuEvent) == 0)) { + ABSL_TSAN_MUTEX_POST_LOCK(this, + __tsan_mutex_read_lock | __tsan_mutex_try_lock | + __tsan_mutex_try_lock_failed, + 0); + return false; + } + // we're recording events + return ReaderTryLockSlow(); +} + +ABSL_ATTRIBUTE_NOINLINE bool Mutex::ReaderTryLockSlow() { + intptr_t v = mu_.load(std::memory_order_relaxed); +#if defined(__clang__) +#pragma nounroll +#endif + for (int loop_limit = 5; loop_limit != 0; loop_limit--) { + if ((v & kShared->slow_need_zero) == 0 && + mu_.compare_exchange_strong(v, (kMuReader | v) + kMuOne, + std::memory_order_acquire, + std::memory_order_relaxed)) { + DebugOnlyLockEnter(this); + PostSynchEvent(this, SYNCH_EV_READERTRYLOCK_SUCCESS); + ABSL_TSAN_MUTEX_POST_LOCK( + this, __tsan_mutex_read_lock | __tsan_mutex_try_lock, 0); + return true; } } + PostSynchEvent(this, SYNCH_EV_READERTRYLOCK_FAILED); ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_read_lock | __tsan_mutex_try_lock | __tsan_mutex_try_lock_failed, @@ -1768,16 +1755,20 @@ void Mutex::ReaderUnlock() { DebugOnlyLockLeave(this); intptr_t v = mu_.load(std::memory_order_relaxed); assert((v & (kMuWriter | kMuReader)) == kMuReader); - if ((v & (kMuReader | kMuWait | kMuEvent)) == kMuReader) { + for (;;) { + if (ABSL_PREDICT_FALSE((v & (kMuReader | kMuWait | kMuEvent)) != + kMuReader)) { + this->UnlockSlow(nullptr /*no waitp*/); // take slow path + break; + } // fast reader release (reader with no waiters) intptr_t clear = ExactlyOneReader(v) ? kMuReader | kMuOne : kMuOne; - if (mu_.compare_exchange_strong(v, v - clear, std::memory_order_release, - std::memory_order_relaxed)) { - ABSL_TSAN_MUTEX_POST_UNLOCK(this, __tsan_mutex_read_lock); - return; + if (ABSL_PREDICT_TRUE( + mu_.compare_exchange_strong(v, v - clear, std::memory_order_release, + std::memory_order_relaxed))) { + break; } } - this->UnlockSlow(nullptr /*no waitp*/); // take slow path ABSL_TSAN_MUTEX_POST_UNLOCK(this, __tsan_mutex_read_lock); } @@ -1812,6 +1803,22 @@ static intptr_t IgnoreWaitingWritersMask(int flag) { // Internal version of LockWhen(). See LockSlowWithDeadline() ABSL_ATTRIBUTE_NOINLINE void Mutex::LockSlow(MuHow how, const Condition* cond, int flags) { + // Note: we specifically initialize spinloop_iterations after the first use + // in TryAcquireWithSpinning so that Lock function does not have any non-tail + // calls and consequently a stack frame. It's fine to have spinloop_iterations + // uninitialized (meaning no spinning) in all initial uncontended Lock calls + // and in the first contended call. After that we will have + // spinloop_iterations properly initialized. + if (ABSL_PREDICT_FALSE( + globals.spinloop_iterations.load(std::memory_order_relaxed) == 0)) { + if (absl::base_internal::NumCPUs() > 1) { + // If this is multiprocessor, allow spinning. + globals.spinloop_iterations.store(1500, std::memory_order_relaxed); + } else { + // If this a uniprocessor, only yield/sleep. + globals.spinloop_iterations.store(-1, std::memory_order_relaxed); + } + } ABSL_RAW_CHECK( this->LockSlowWithDeadline(how, cond, KernelTimeout::Never(), flags), "condition untrue on return from LockSlow"); @@ -1913,7 +1920,7 @@ bool Mutex::LockSlowWithDeadline(MuHow how, const Condition* cond, SynchWaitParams waitp(how, cond, t, nullptr /*no cvmu*/, Synch_GetPerThreadAnnotated(this), nullptr /*no cv_word*/); - if (!Condition::GuaranteedEqual(cond, nullptr)) { + if (cond != nullptr) { flags |= kMuIsCond; } if (unlock) { @@ -2103,7 +2110,6 @@ ABSL_ATTRIBUTE_NOINLINE void Mutex::UnlockSlow(SynchWaitParams* waitp) { // head of the list searched previously, or zero PerThreadSynch* old_h = nullptr; // a condition that's known to be false. - const Condition* known_false = nullptr; PerThreadSynch* wake_list = kPerThreadSynchNull; // list of threads to wake intptr_t wr_wait = 0; // set to kMuWrWait if we wake a reader and a // later writer could have acquired the lock @@ -2207,7 +2213,7 @@ ABSL_ATTRIBUTE_NOINLINE void Mutex::UnlockSlow(SynchWaitParams* waitp) { } } if (h->next->waitp->how == kExclusive && - Condition::GuaranteedEqual(h->next->waitp->cond, nullptr)) { + h->next->waitp->cond == nullptr) { // easy case: writer with no condition; no need to search pw = h; // wake w, the successor of h (=pw) w = h->next; @@ -2290,10 +2296,8 @@ ABSL_ATTRIBUTE_NOINLINE void Mutex::UnlockSlow(SynchWaitParams* waitp) { w_walk->wake = false; if (w_walk->waitp->cond == nullptr || // no condition => vacuously true OR - (w_walk->waitp->cond != known_false && - // this thread's condition is not known false, AND - // is in fact true - EvalConditionIgnored(this, w_walk->waitp->cond))) { + // this thread's condition is true + EvalConditionIgnored(this, w_walk->waitp->cond)) { if (w == nullptr) { w_walk->wake = true; // can wake this waiter w = w_walk; @@ -2307,8 +2311,6 @@ ABSL_ATTRIBUTE_NOINLINE void Mutex::UnlockSlow(SynchWaitParams* waitp) { } else { // writer with true condition wr_wait = kMuWrWait; } - } else { // can't wake; condition false - known_false = w_walk->waitp->cond; // remember last false condition } if (w_walk->wake) { // we're waking reader w_walk pw_walk = w_walk; // don't skip similar waiters @@ -2414,10 +2416,10 @@ void Mutex::Fer(PerThreadSynch* w) { int c = 0; ABSL_RAW_CHECK(w->waitp->cond == nullptr, "Mutex::Fer while waiting on Condition"); - ABSL_RAW_CHECK(!w->waitp->timeout.has_timeout(), - "Mutex::Fer while in timed wait"); ABSL_RAW_CHECK(w->waitp->cv_word == nullptr, "Mutex::Fer with pending CondVar queueing"); + // The CondVar timeout is not relevant for the Mutex wait. + w->waitp->timeout = {}; for (;;) { intptr_t v = mu_.load(std::memory_order_relaxed); // Note: must not queue if the mutex is unlocked (nobody will wake it). @@ -2436,7 +2438,8 @@ void Mutex::Fer(PerThreadSynch* w) { } else { if ((v & (kMuSpin | kMuWait)) == 0) { // no waiters // This thread tries to become the one and only waiter. - PerThreadSynch* new_h = Enqueue(nullptr, w->waitp, v, kMuIsCond); + PerThreadSynch* new_h = + Enqueue(nullptr, w->waitp, v, kMuIsCond | kMuIsFer); ABSL_RAW_CHECK(new_h != nullptr, "Enqueue failed"); // we must queue ourselves if (mu_.compare_exchange_strong( @@ -2447,7 +2450,7 @@ void Mutex::Fer(PerThreadSynch* w) { } else if ((v & kMuSpin) == 0 && mu_.compare_exchange_strong(v, v | kMuSpin | kMuWait)) { PerThreadSynch* h = GetPerThreadSynch(v); - PerThreadSynch* new_h = Enqueue(h, w->waitp, v, kMuIsCond); + PerThreadSynch* new_h = Enqueue(h, w->waitp, v, kMuIsCond | kMuIsFer); ABSL_RAW_CHECK(new_h != nullptr, "Enqueue failed"); // we must queue ourselves do { @@ -2503,12 +2506,6 @@ void CondVar::EnableDebugLog(const char* name) { UnrefSynchEvent(e); } -CondVar::~CondVar() { - if ((cv_.load(std::memory_order_relaxed) & kCvEvent) != 0) { - ForgetSynchEvent(&this->cv_, kCvEvent, kCvSpin); - } -} - // Remove thread s from the list of waiters on this condition variable. void CondVar::Remove(PerThreadSynch* s) { SchedulingGuard::ScopedDisable disable_rescheduling; @@ -2659,33 +2656,6 @@ bool CondVar::WaitCommon(Mutex* mutex, KernelTimeout t) { return rc; } -bool CondVar::WaitWithTimeout(Mutex* mu, absl::Duration timeout) { - return WaitCommon(mu, KernelTimeout(timeout)); -} - -bool CondVar::WaitWithDeadline(Mutex* mu, absl::Time deadline) { - return WaitCommon(mu, KernelTimeout(deadline)); -} - -void CondVar::Wait(Mutex* mu) { WaitCommon(mu, KernelTimeout::Never()); } - -// Wake thread w -// If it was a timed wait, w will be waiting on w->cv -// Otherwise, if it was not a Mutex mutex, w will be waiting on w->sem -// Otherwise, w is transferred to the Mutex mutex via Mutex::Fer(). -void CondVar::Wakeup(PerThreadSynch* w) { - if (w->waitp->timeout.has_timeout() || w->waitp->cvmu == nullptr) { - // The waiting thread only needs to observe "w->state == kAvailable" to be - // released, we must cache "cvmu" before clearing "next". - Mutex* mu = w->waitp->cvmu; - w->next = nullptr; - w->state.store(PerThreadSynch::kAvailable, std::memory_order_release); - Mutex::IncrementSynchSem(mu, w); - } else { - w->waitp->cvmu->Fer(w); - } -} - void CondVar::Signal() { SchedulingGuard::ScopedDisable disable_rescheduling; ABSL_TSAN_MUTEX_PRE_SIGNAL(nullptr, 0); @@ -2710,7 +2680,7 @@ void CondVar::Signal() { cv_.store((v & kCvEvent) | reinterpret_cast<intptr_t>(h), std::memory_order_release); if (w != nullptr) { - CondVar::Wakeup(w); // wake waiter, if there was one + w->waitp->cvmu->Fer(w); // wake waiter, if there was one cond_var_tracer("Signal wakeup", this); } if ((v & kCvEvent) != 0) { @@ -2746,7 +2716,7 @@ void CondVar::SignalAll() { do { // for every thread, wake it up w = n; n = n->next; - CondVar::Wakeup(w); + w->waitp->cvmu->Fer(w); } while (w != h); cond_var_tracer("SignalAll wakeup", this); } @@ -2810,17 +2780,11 @@ Condition::Condition(const bool* cond) StoreCallback(dereference); } -bool Condition::Eval() const { - // eval_ == null for kTrue - return (this->eval_ == nullptr) || (*this->eval_)(this); -} +bool Condition::Eval() const { return (*this->eval_)(this); } bool Condition::GuaranteedEqual(const Condition* a, const Condition* b) { - // kTrue logic. - if (a == nullptr || a->eval_ == nullptr) { - return b == nullptr || b->eval_ == nullptr; - } else if (b == nullptr || b->eval_ == nullptr) { - return false; + if (a == nullptr || b == nullptr) { + return a == b; } // Check equality of the representative fields. return a->eval_ == b->eval_ && a->arg_ == b->arg_ && diff --git a/contrib/restricted/abseil-cpp/absl/synchronization/mutex.h b/contrib/restricted/abseil-cpp/absl/synchronization/mutex.h index 645c26d9c3..d53a22bbd9 100644 --- a/contrib/restricted/abseil-cpp/absl/synchronization/mutex.h +++ b/contrib/restricted/abseil-cpp/absl/synchronization/mutex.h @@ -64,6 +64,7 @@ #include <iterator> #include <string> +#include "absl/base/attributes.h" #include "absl/base/const_init.h" #include "absl/base/internal/identity.h" #include "absl/base/internal/low_level_alloc.h" @@ -324,7 +325,9 @@ class ABSL_LOCKABLE Mutex { // `true`, `Await()` *may* skip the release/re-acquire step. // // `Await()` requires that this thread holds this `Mutex` in some mode. - void Await(const Condition& cond); + void Await(const Condition& cond) { + AwaitCommon(cond, synchronization_internal::KernelTimeout::Never()); + } // Mutex::LockWhen() // Mutex::ReaderLockWhen() @@ -334,9 +337,15 @@ class ABSL_LOCKABLE Mutex { // be acquired, then atomically acquires this `Mutex`. `LockWhen()` is // logically equivalent to `*Lock(); Await();` though they may have different // performance characteristics. - void LockWhen(const Condition& cond) ABSL_EXCLUSIVE_LOCK_FUNCTION(); + void LockWhen(const Condition& cond) ABSL_EXCLUSIVE_LOCK_FUNCTION() { + LockWhenCommon(cond, synchronization_internal::KernelTimeout::Never(), + true); + } - void ReaderLockWhen(const Condition& cond) ABSL_SHARED_LOCK_FUNCTION(); + void ReaderLockWhen(const Condition& cond) ABSL_SHARED_LOCK_FUNCTION() { + LockWhenCommon(cond, synchronization_internal::KernelTimeout::Never(), + false); + } void WriterLockWhen(const Condition& cond) ABSL_EXCLUSIVE_LOCK_FUNCTION() { this->LockWhen(cond); @@ -363,9 +372,13 @@ class ABSL_LOCKABLE Mutex { // Negative timeouts are equivalent to a zero timeout. // // This method requires that this thread holds this `Mutex` in some mode. - bool AwaitWithTimeout(const Condition& cond, absl::Duration timeout); + bool AwaitWithTimeout(const Condition& cond, absl::Duration timeout) { + return AwaitCommon(cond, synchronization_internal::KernelTimeout{timeout}); + } - bool AwaitWithDeadline(const Condition& cond, absl::Time deadline); + bool AwaitWithDeadline(const Condition& cond, absl::Time deadline) { + return AwaitCommon(cond, synchronization_internal::KernelTimeout{deadline}); + } // Mutex::LockWhenWithTimeout() // Mutex::ReaderLockWhenWithTimeout() @@ -379,9 +392,15 @@ class ABSL_LOCKABLE Mutex { // // Negative timeouts are equivalent to a zero timeout. bool LockWhenWithTimeout(const Condition& cond, absl::Duration timeout) - ABSL_EXCLUSIVE_LOCK_FUNCTION(); + ABSL_EXCLUSIVE_LOCK_FUNCTION() { + return LockWhenCommon( + cond, synchronization_internal::KernelTimeout{timeout}, true); + } bool ReaderLockWhenWithTimeout(const Condition& cond, absl::Duration timeout) - ABSL_SHARED_LOCK_FUNCTION(); + ABSL_SHARED_LOCK_FUNCTION() { + return LockWhenCommon( + cond, synchronization_internal::KernelTimeout{timeout}, false); + } bool WriterLockWhenWithTimeout(const Condition& cond, absl::Duration timeout) ABSL_EXCLUSIVE_LOCK_FUNCTION() { return this->LockWhenWithTimeout(cond, timeout); @@ -399,9 +418,15 @@ class ABSL_LOCKABLE Mutex { // // Deadlines in the past are equivalent to an immediate deadline. bool LockWhenWithDeadline(const Condition& cond, absl::Time deadline) - ABSL_EXCLUSIVE_LOCK_FUNCTION(); + ABSL_EXCLUSIVE_LOCK_FUNCTION() { + return LockWhenCommon( + cond, synchronization_internal::KernelTimeout{deadline}, true); + } bool ReaderLockWhenWithDeadline(const Condition& cond, absl::Time deadline) - ABSL_SHARED_LOCK_FUNCTION(); + ABSL_SHARED_LOCK_FUNCTION() { + return LockWhenCommon( + cond, synchronization_internal::KernelTimeout{deadline}, false); + } bool WriterLockWhenWithDeadline(const Condition& cond, absl::Time deadline) ABSL_EXCLUSIVE_LOCK_FUNCTION() { return this->LockWhenWithDeadline(cond, deadline); @@ -497,15 +522,22 @@ class ABSL_LOCKABLE Mutex { int flags) ABSL_ATTRIBUTE_COLD; // slow path release void UnlockSlow(SynchWaitParams* waitp) ABSL_ATTRIBUTE_COLD; + // TryLock slow path. + bool TryLockSlow(); + // ReaderTryLock slow path. + bool ReaderTryLockSlow(); // Common code between Await() and AwaitWithTimeout/Deadline() bool AwaitCommon(const Condition& cond, synchronization_internal::KernelTimeout t); + bool LockWhenCommon(const Condition& cond, + synchronization_internal::KernelTimeout t, bool write); // Attempt to remove thread s from queue. void TryRemove(base_internal::PerThreadSynch* s); // Block a thread on mutex. void Block(base_internal::PerThreadSynch* s); // Wake a thread; return successor. base_internal::PerThreadSynch* Wakeup(base_internal::PerThreadSynch* w); + void Dtor(); friend class CondVar; // for access to Trans()/Fer(). void Trans(MuHow how); // used for CondVar->Mutex transfer @@ -708,23 +740,25 @@ class Condition { // a function template is passed as `func`. Also, the dummy `typename = void` // template parameter exists just to work around a MSVC mangling bug. template <typename T, typename = void> - Condition(bool (*func)(T*), typename absl::internal::identity<T>::type* arg); + Condition(bool (*func)(T*), + typename absl::internal::type_identity<T>::type* arg); // Templated version for invoking a method that returns a `bool`. // // `Condition(object, &Class::Method)` constructs a `Condition` that evaluates // `object->Method()`. // - // Implementation Note: `absl::internal::identity` is used to allow methods to - // come from base classes. A simpler signature like + // Implementation Note: `absl::internal::type_identity` is used to allow + // methods to come from base classes. A simpler signature like // `Condition(T*, bool (T::*)())` does not suffice. template <typename T> - Condition(T* object, bool (absl::internal::identity<T>::type::*method)()); + Condition(T* object, + bool (absl::internal::type_identity<T>::type::*method)()); // Same as above, for const members template <typename T> Condition(const T* object, - bool (absl::internal::identity<T>::type::*method)() const); + bool (absl::internal::type_identity<T>::type::*method)() const); // A Condition that returns the value of `*cond` explicit Condition(const bool* cond); @@ -816,7 +850,7 @@ class Condition { static bool CallVoidPtrFunction(const Condition*); template <typename T> static bool CastAndCallFunction(const Condition* c); - template <typename T> + template <typename T, typename ConditionMethodPtr> static bool CastAndCallMethod(const Condition* c); // Helper methods for storing, validating, and reading callback arguments. @@ -833,8 +867,10 @@ class Condition { std::memcpy(callback, callback_, sizeof(*callback)); } + static bool AlwaysTrue(const Condition*) { return true; } + // Used only to create kTrue. - constexpr Condition() = default; + constexpr Condition() : eval_(AlwaysTrue), arg_(nullptr) {} }; // ----------------------------------------------------------------------------- @@ -877,7 +913,6 @@ class CondVar { // A `CondVar` allocated on the heap or on the stack can use the this // constructor. CondVar(); - ~CondVar(); // CondVar::Wait() // @@ -886,7 +921,9 @@ class CondVar { // spurious wakeup), then reacquires the `Mutex` and returns. // // Requires and ensures that the current thread holds the `Mutex`. - void Wait(Mutex* mu); + void Wait(Mutex* mu) { + WaitCommon(mu, synchronization_internal::KernelTimeout::Never()); + } // CondVar::WaitWithTimeout() // @@ -901,7 +938,9 @@ class CondVar { // to return `true` or `false`. // // Requires and ensures that the current thread holds the `Mutex`. - bool WaitWithTimeout(Mutex* mu, absl::Duration timeout); + bool WaitWithTimeout(Mutex* mu, absl::Duration timeout) { + return WaitCommon(mu, synchronization_internal::KernelTimeout(timeout)); + } // CondVar::WaitWithDeadline() // @@ -918,7 +957,9 @@ class CondVar { // to return `true` or `false`. // // Requires and ensures that the current thread holds the `Mutex`. - bool WaitWithDeadline(Mutex* mu, absl::Time deadline); + bool WaitWithDeadline(Mutex* mu, absl::Time deadline) { + return WaitCommon(mu, synchronization_internal::KernelTimeout(deadline)); + } // CondVar::Signal() // @@ -940,7 +981,6 @@ class CondVar { private: bool WaitCommon(Mutex* mutex, synchronization_internal::KernelTimeout t); void Remove(base_internal::PerThreadSynch* s); - void Wakeup(base_internal::PerThreadSynch* w); std::atomic<intptr_t> cv_; // Condition variable state. CondVar(const CondVar&) = delete; CondVar& operator=(const CondVar&) = delete; @@ -1024,15 +1064,30 @@ inline Mutex::Mutex() : mu_(0) { inline constexpr Mutex::Mutex(absl::ConstInitType) : mu_(0) {} +#if !defined(__APPLE__) && !defined(ABSL_BUILD_DLL) +ABSL_ATTRIBUTE_ALWAYS_INLINE +inline Mutex::~Mutex() { Dtor(); } +#endif + +#if defined(NDEBUG) && !defined(ABSL_HAVE_THREAD_SANITIZER) +// Use default (empty) destructor in release build for performance reasons. +// We need to mark both Dtor and ~Mutex as always inline for inconsistent +// builds that use both NDEBUG and !NDEBUG with dynamic libraries. In these +// cases we want the empty functions to dissolve entirely rather than being +// exported from dynamic libraries and potentially override the non-empty ones. +ABSL_ATTRIBUTE_ALWAYS_INLINE +inline void Mutex::Dtor() {} +#endif + inline CondVar::CondVar() : cv_(0) {} // static -template <typename T> +template <typename T, typename ConditionMethodPtr> bool Condition::CastAndCallMethod(const Condition* c) { T* object = static_cast<T*>(c->arg_); - bool (T::*method_pointer)(); - c->ReadCallback(&method_pointer); - return (object->*method_pointer)(); + ConditionMethodPtr condition_method_pointer; + c->ReadCallback(&condition_method_pointer); + return (object->*condition_method_pointer)(); } // static @@ -1054,25 +1109,25 @@ inline Condition::Condition(bool (*func)(T*), T* arg) } template <typename T, typename> -inline Condition::Condition(bool (*func)(T*), - typename absl::internal::identity<T>::type* arg) +inline Condition::Condition( + bool (*func)(T*), typename absl::internal::type_identity<T>::type* arg) // Just delegate to the overload above. : Condition(func, arg) {} template <typename T> -inline Condition::Condition(T* object, - bool (absl::internal::identity<T>::type::*method)()) - : eval_(&CastAndCallMethod<T>), arg_(object) { +inline Condition::Condition( + T* object, bool (absl::internal::type_identity<T>::type::*method)()) + : eval_(&CastAndCallMethod<T, decltype(method)>), arg_(object) { static_assert(sizeof(&method) <= sizeof(callback_), "An overlarge method pointer was passed to Condition."); StoreCallback(method); } template <typename T> -inline Condition::Condition(const T* object, - bool (absl::internal::identity<T>::type::*method)() - const) - : eval_(&CastAndCallMethod<T>), +inline Condition::Condition( + const T* object, + bool (absl::internal::type_identity<T>::type::*method)() const) + : eval_(&CastAndCallMethod<const T, decltype(method)>), arg_(reinterpret_cast<void*>(const_cast<T*>(object))) { StoreCallback(method); } diff --git a/contrib/restricted/abseil-cpp/absl/time/civil_time.h b/contrib/restricted/abseil-cpp/absl/time/civil_time.h index 5855bc73a1..3e904a113d 100644 --- a/contrib/restricted/abseil-cpp/absl/time/civil_time.h +++ b/contrib/restricted/abseil-cpp/absl/time/civil_time.h @@ -462,6 +462,32 @@ std::string FormatCivilTime(CivilDay c); std::string FormatCivilTime(CivilMonth c); std::string FormatCivilTime(CivilYear c); +// Support for StrFormat(), StrCat(), etc +template <typename Sink> +void AbslStringify(Sink& sink, CivilSecond c) { + sink.Append(FormatCivilTime(c)); +} +template <typename Sink> +void AbslStringify(Sink& sink, CivilMinute c) { + sink.Append(FormatCivilTime(c)); +} +template <typename Sink> +void AbslStringify(Sink& sink, CivilHour c) { + sink.Append(FormatCivilTime(c)); +} +template <typename Sink> +void AbslStringify(Sink& sink, CivilDay c) { + sink.Append(FormatCivilTime(c)); +} +template <typename Sink> +void AbslStringify(Sink& sink, CivilMonth c) { + sink.Append(FormatCivilTime(c)); +} +template <typename Sink> +void AbslStringify(Sink& sink, CivilYear c) { + sink.Append(FormatCivilTime(c)); +} + // absl::ParseCivilTime() // // Parses a civil-time value from the specified `absl::string_view` into the diff --git a/contrib/restricted/abseil-cpp/absl/time/clock.h b/contrib/restricted/abseil-cpp/absl/time/clock.h index 5fe244d638..41d2cf2770 100644 --- a/contrib/restricted/abseil-cpp/absl/time/clock.h +++ b/contrib/restricted/abseil-cpp/absl/time/clock.h @@ -22,6 +22,9 @@ #ifndef ABSL_TIME_CLOCK_H_ #define ABSL_TIME_CLOCK_H_ +#include <cstdint> + +#include "absl/base/config.h" #include "absl/base/macros.h" #include "absl/time/time.h" @@ -64,7 +67,8 @@ ABSL_NAMESPACE_END // By changing our extension points to be extern "C", we dodge this // check. extern "C" { -void ABSL_INTERNAL_C_SYMBOL(AbslInternalSleepFor)(absl::Duration duration); +ABSL_DLL void ABSL_INTERNAL_C_SYMBOL(AbslInternalSleepFor)( + absl::Duration duration); } // extern "C" inline void absl::SleepFor(absl::Duration duration) { diff --git a/contrib/restricted/abseil-cpp/absl/time/duration.cc b/contrib/restricted/abseil-cpp/absl/time/duration.cc index 634e5d5845..bdb16e2109 100644 --- a/contrib/restricted/abseil-cpp/absl/time/duration.cc +++ b/contrib/restricted/abseil-cpp/absl/time/duration.cc @@ -55,8 +55,7 @@ #include <algorithm> #include <cassert> -#include <cctype> -#include <cerrno> +#include <chrono> // NOLINT(build/c++11) #include <cmath> #include <cstdint> #include <cstdlib> @@ -66,8 +65,9 @@ #include <limits> #include <string> +#include "absl/base/attributes.h" #include "absl/base/casts.h" -#include "absl/base/macros.h" +#include "absl/base/config.h" #include "absl/numeric/int128.h" #include "absl/strings/string_view.h" #include "absl/strings/strip.h" diff --git a/contrib/restricted/abseil-cpp/absl/time/internal/cctz/include/cctz/civil_time_detail.h b/contrib/restricted/abseil-cpp/absl/time/internal/cctz/include/cctz/civil_time_detail.h index a5b084e6be..2b0aed56c3 100644 --- a/contrib/restricted/abseil-cpp/absl/time/internal/cctz/include/cctz/civil_time_detail.h +++ b/contrib/restricted/abseil-cpp/absl/time/internal/cctz/include/cctz/civil_time_detail.h @@ -401,11 +401,11 @@ class civil_time { : civil_time(ct.f_) {} // Factories for the maximum/minimum representable civil_time. - static CONSTEXPR_F civil_time(max)() { + static CONSTEXPR_F auto(max)() -> civil_time { const auto max_year = (std::numeric_limits<std::int_least64_t>::max)(); return civil_time(max_year, 12, 31, 23, 59, 59); } - static CONSTEXPR_F civil_time(min)() { + static CONSTEXPR_F auto(min)() -> civil_time { const auto min_year = (std::numeric_limits<std::int_least64_t>::min)(); return civil_time(min_year, 1, 1, 0, 0, 0); } diff --git a/contrib/restricted/abseil-cpp/absl/time/internal/cctz/src/time_zone_format.cc b/contrib/restricted/abseil-cpp/absl/time/internal/cctz/src/time_zone_format.cc index 9b91f61cf0..e7e30a2fb7 100644 --- a/contrib/restricted/abseil-cpp/absl/time/internal/cctz/src/time_zone_format.cc +++ b/contrib/restricted/abseil-cpp/absl/time/internal/cctz/src/time_zone_format.cc @@ -19,7 +19,7 @@ #endif #if defined(HAS_STRPTIME) && HAS_STRPTIME -#if !defined(_XOPEN_SOURCE) && !defined(__OpenBSD__) +#if !defined(_XOPEN_SOURCE) && !defined(__FreeBSD__) && !defined(__OpenBSD__) #define _XOPEN_SOURCE 500 // Exposes definitions for SUSv2 (UNIX 98). #endif #endif diff --git a/contrib/restricted/abseil-cpp/absl/time/internal/cctz/src/time_zone_info.cc b/contrib/restricted/abseil-cpp/absl/time/internal/cctz/src/time_zone_info.cc index f46198ffaa..b7178a6b67 100644 --- a/contrib/restricted/abseil-cpp/absl/time/internal/cctz/src/time_zone_info.cc +++ b/contrib/restricted/abseil-cpp/absl/time/internal/cctz/src/time_zone_info.cc @@ -474,7 +474,8 @@ std::unique_ptr<ZoneInfoSource> AndroidZoneInfoSource::Open( const std::size_t pos = (name.compare(0, 5, "file:") == 0) ? 5 : 0; // See Android's libc/tzcode/bionic.cpp for additional information. - for (const char* tzdata : {"/data/misc/zoneinfo/current/tzdata", + for (const char* tzdata : {"/apex/com.android.tzdata/etc/tz/tzdata", + "/data/misc/zoneinfo/current/tzdata", "/system/usr/share/zoneinfo/tzdata"}) { auto fp = FOpen(tzdata, "rb"); if (fp == nullptr) continue; @@ -539,9 +540,16 @@ std::unique_ptr<ZoneInfoSource> FuchsiaZoneInfoSource::Open( // Prefixes where a Fuchsia component might find zoneinfo files, // in descending order of preference. const auto kTzdataPrefixes = { + // The tzdata from `config-data`. "/config/data/tzdata/", + // The tzdata bundled in the component's package. "/pkg/data/tzdata/", + // General data storage. "/data/tzdata/", + // The recommended path for routed-in tzdata files. + // See for details: + // https://fuchsia.dev/fuchsia-src/concepts/process/namespaces?hl=en#typical_directory_structure + "/config/tzdata/", }; const auto kEmptyPrefix = {""}; const bool name_absolute = (pos != name.size() && name[pos] == '/'); @@ -744,19 +752,6 @@ bool TimeZoneInfo::Load(ZoneInfoSource* zip) { version_ = zip->Version(); } - // Trim redundant transitions. zic may have added these to work around - // differences between the glibc and reference implementations (see - // zic.c:dontmerge) or to avoid bugs in old readers. For us, they just - // get in the way when we do future_spec_ extension. - while (hdr.timecnt > 1) { - if (!EquivTransitions(transitions_[hdr.timecnt - 1].type_index, - transitions_[hdr.timecnt - 2].type_index)) { - break; - } - hdr.timecnt -= 1; - } - transitions_.resize(hdr.timecnt); - // Ensure that there is always a transition in the first half of the // time line (the second half is handled below) so that the signed // difference between a civil_second and the civil_second of its diff --git a/contrib/restricted/abseil-cpp/absl/time/internal/cctz/src/tzfile.h b/contrib/restricted/abseil-cpp/absl/time/internal/cctz/src/tzfile.h index 9613055db7..114026d066 100644 --- a/contrib/restricted/abseil-cpp/absl/time/internal/cctz/src/tzfile.h +++ b/contrib/restricted/abseil-cpp/absl/time/internal/cctz/src/tzfile.h @@ -21,14 +21,6 @@ ** Information about time zone files. */ -#ifndef TZDIR -#define TZDIR "/usr/share/zoneinfo" /* Time zone object file directory */ -#endif /* !defined TZDIR */ - -#ifndef TZDEFAULT -#define TZDEFAULT "/etc/localtime" -#endif /* !defined TZDEFAULT */ - #ifndef TZDEFRULES #define TZDEFRULES "posixrules" #endif /* !defined TZDEFRULES */ diff --git a/contrib/restricted/abseil-cpp/absl/types/bad_any_cast.cc b/contrib/restricted/abseil-cpp/absl/types/bad_any_cast.cc index b0592cc9bc..22558b48c2 100644 --- a/contrib/restricted/abseil-cpp/absl/types/bad_any_cast.cc +++ b/contrib/restricted/abseil-cpp/absl/types/bad_any_cast.cc @@ -43,4 +43,22 @@ void ThrowBadAnyCast() { ABSL_NAMESPACE_END } // namespace absl +#else + +// https://github.com/abseil/abseil-cpp/issues/1465 +// CMake builds on Apple platforms error when libraries are empty. +// Our CMake configuration can avoid this error on header-only libraries, +// but since this library is conditionally empty, including a single +// variable is an easy workaround. +#ifdef __APPLE__ +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace types_internal { +extern const char kAvoidEmptyBadAnyCastLibraryWarning; +const char kAvoidEmptyBadAnyCastLibraryWarning = 0; +} // namespace types_internal +ABSL_NAMESPACE_END +} // namespace absl +#endif // __APPLE__ + #endif // ABSL_USES_STD_ANY diff --git a/contrib/restricted/abseil-cpp/absl/types/bad_optional_access.cc b/contrib/restricted/abseil-cpp/absl/types/bad_optional_access.cc index 26aca70d9c..2552cc853a 100644 --- a/contrib/restricted/abseil-cpp/absl/types/bad_optional_access.cc +++ b/contrib/restricted/abseil-cpp/absl/types/bad_optional_access.cc @@ -45,4 +45,22 @@ void throw_bad_optional_access() { ABSL_NAMESPACE_END } // namespace absl +#else + +// https://github.com/abseil/abseil-cpp/issues/1465 +// CMake builds on Apple platforms error when libraries are empty. +// Our CMake configuration can avoid this error on header-only libraries, +// but since this library is conditionally empty, including a single +// variable is an easy workaround. +#ifdef __APPLE__ +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace types_internal { +extern const char kAvoidEmptyBadOptionalAccessLibraryWarning; +const char kAvoidEmptyBadOptionalAccessLibraryWarning = 0; +} // namespace types_internal +ABSL_NAMESPACE_END +} // namespace absl +#endif // __APPLE__ + #endif // ABSL_USES_STD_OPTIONAL diff --git a/contrib/restricted/abseil-cpp/absl/types/bad_variant_access.cc b/contrib/restricted/abseil-cpp/absl/types/bad_variant_access.cc index 3dc88cc09f..a76aa80dfa 100644 --- a/contrib/restricted/abseil-cpp/absl/types/bad_variant_access.cc +++ b/contrib/restricted/abseil-cpp/absl/types/bad_variant_access.cc @@ -61,4 +61,22 @@ void Rethrow() { ABSL_NAMESPACE_END } // namespace absl +#else + +// https://github.com/abseil/abseil-cpp/issues/1465 +// CMake builds on Apple platforms error when libraries are empty. +// Our CMake configuration can avoid this error on header-only libraries, +// but since this library is conditionally empty, including a single +// variable is an easy workaround. +#ifdef __APPLE__ +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace types_internal { +extern const char kAvoidEmptyBadVariantAccessLibraryWarning; +const char kAvoidEmptyBadVariantAccessLibraryWarning = 0; +} // namespace types_internal +ABSL_NAMESPACE_END +} // namespace absl +#endif // __APPLE__ + #endif // ABSL_USES_STD_VARIANT diff --git a/contrib/restricted/abseil-cpp/absl/types/internal/variant.h b/contrib/restricted/abseil-cpp/absl/types/internal/variant.h index fc8829e5d8..263d7b0947 100644 --- a/contrib/restricted/abseil-cpp/absl/types/internal/variant.h +++ b/contrib/restricted/abseil-cpp/absl/types/internal/variant.h @@ -1047,11 +1047,11 @@ class VariantStateBase { std::size_t index_; }; -using absl::internal::identity; +using absl::internal::type_identity; // OverloadSet::Overload() is a unary function which is overloaded to // take any of the element types of the variant, by reference-to-const. -// The return type of the overload on T is identity<T>, so that you +// The return type of the overload on T is type_identity<T>, so that you // can statically determine which overload was called. // // Overload() is not defined, so it can only be called in unevaluated @@ -1062,7 +1062,7 @@ struct OverloadSet; template <typename T, typename... Ts> struct OverloadSet<T, Ts...> : OverloadSet<Ts...> { using Base = OverloadSet<Ts...>; - static identity<T> Overload(const T&); + static type_identity<T> Overload(const T&); using Base::Overload; }; diff --git a/contrib/restricted/abseil-cpp/absl/types/optional.h b/contrib/restricted/abseil-cpp/absl/types/optional.h index 0a8080dc1a..395fe62f29 100644 --- a/contrib/restricted/abseil-cpp/absl/types/optional.h +++ b/contrib/restricted/abseil-cpp/absl/types/optional.h @@ -61,6 +61,7 @@ ABSL_NAMESPACE_END #include <utility> #include "absl/base/attributes.h" +#include "absl/base/nullability.h" #include "absl/base/internal/inline_variable.h" #include "absl/meta/type_traits.h" #include "absl/types/bad_optional_access.h" @@ -415,11 +416,11 @@ class optional : private optional_internal::optional_data<T>, // `optional` is empty, behavior is undefined. // // If you need myOpt->foo in constexpr, use (*myOpt).foo instead. - const T* operator->() const ABSL_ATTRIBUTE_LIFETIME_BOUND { + absl::Nonnull<const T*> operator->() const ABSL_ATTRIBUTE_LIFETIME_BOUND { ABSL_HARDENING_ASSERT(this->engaged_); return std::addressof(this->data_); } - T* operator->() ABSL_ATTRIBUTE_LIFETIME_BOUND { + absl::Nonnull<T*> operator->() ABSL_ATTRIBUTE_LIFETIME_BOUND { ABSL_HARDENING_ASSERT(this->engaged_); return std::addressof(this->data_); } diff --git a/contrib/restricted/abseil-cpp/absl/types/span.h b/contrib/restricted/abseil-cpp/absl/types/span.h index 70ed8eb6fa..88cd759540 100644 --- a/contrib/restricted/abseil-cpp/absl/types/span.h +++ b/contrib/restricted/abseil-cpp/absl/types/span.h @@ -63,6 +63,7 @@ #include "absl/base/attributes.h" #include "absl/base/internal/throw_delegate.h" #include "absl/base/macros.h" +#include "absl/base/nullability.h" #include "absl/base/optimization.h" #include "absl/base/port.h" // TODO(strel): remove this include #include "absl/meta/type_traits.h" @@ -172,6 +173,8 @@ class Span { public: using element_type = T; using value_type = absl::remove_cv_t<T>; + // TODO(b/316099902) - pointer should be Nullable<T*>, but this makes it hard + // to recognize foreach loops as safe. using pointer = T*; using const_pointer = const T*; using reference = T&; @@ -679,12 +682,12 @@ bool operator>=(Span<T> a, const U& b) { // } // template <int&... ExplicitArgumentBarrier, typename T> -constexpr Span<T> MakeSpan(T* ptr, size_t size) noexcept { +constexpr Span<T> MakeSpan(absl::Nullable<T*> ptr, size_t size) noexcept { return Span<T>(ptr, size); } template <int&... ExplicitArgumentBarrier, typename T> -Span<T> MakeSpan(T* begin, T* end) noexcept { +Span<T> MakeSpan(absl::Nullable<T*> begin, absl::Nullable<T*> end) noexcept { return ABSL_HARDENING_ASSERT(begin <= end), Span<T>(begin, static_cast<size_t>(end - begin)); } @@ -725,12 +728,14 @@ constexpr Span<T> MakeSpan(T (&array)[N]) noexcept { // ProcessInts(absl::MakeConstSpan(std::vector<int>{ 0, 0, 0 })); // template <int&... ExplicitArgumentBarrier, typename T> -constexpr Span<const T> MakeConstSpan(T* ptr, size_t size) noexcept { +constexpr Span<const T> MakeConstSpan(absl::Nullable<T*> ptr, + size_t size) noexcept { return Span<const T>(ptr, size); } template <int&... ExplicitArgumentBarrier, typename T> -Span<const T> MakeConstSpan(T* begin, T* end) noexcept { +Span<const T> MakeConstSpan(absl::Nullable<T*> begin, + absl::Nullable<T*> end) noexcept { return ABSL_HARDENING_ASSERT(begin <= end), Span<const T>(begin, end - begin); } diff --git a/contrib/restricted/abseil-cpp/absl/utility/utility.h b/contrib/restricted/abseil-cpp/absl/utility/utility.h index bf9232209a..fc0d1f6551 100644 --- a/contrib/restricted/abseil-cpp/absl/utility/utility.h +++ b/contrib/restricted/abseil-cpp/absl/utility/utility.h @@ -12,17 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. // -// This header file contains C++11 versions of standard <utility> header -// abstractions available within C++14 and C++17, and are designed to be drop-in +// This header file contains C++14 versions of standard <utility> header +// abstractions available within C++17, and are designed to be drop-in // replacement for code compliant with C++14 and C++17. // // The following abstractions are defined: // -// * integer_sequence<T, Ints...> == std::integer_sequence<T, Ints...> -// * index_sequence<Ints...> == std::index_sequence<Ints...> -// * make_integer_sequence<T, N> == std::make_integer_sequence<T, N> -// * make_index_sequence<N> == std::make_index_sequence<N> -// * index_sequence_for<Ts...> == std::index_sequence_for<Ts...> // * apply<Functor, Tuple> == std::apply<Functor, Tuple> // * exchange<T> == std::exchange<T> // * make_from_tuple<T> == std::make_from_tuple<T> @@ -33,7 +28,6 @@ // // References: // -// https://en.cppreference.com/w/cpp/utility/integer_sequence // https://en.cppreference.com/w/cpp/utility/apply // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2013/n3658.html @@ -53,68 +47,18 @@ namespace absl { ABSL_NAMESPACE_BEGIN -// integer_sequence -// -// Class template representing a compile-time integer sequence. An instantiation -// of `integer_sequence<T, Ints...>` has a sequence of integers encoded in its -// type through its template arguments (which is a common need when -// working with C++11 variadic templates). `absl::integer_sequence` is designed -// to be a drop-in replacement for C++14's `std::integer_sequence`. -// -// Example: -// -// template< class T, T... Ints > -// void user_function(integer_sequence<T, Ints...>); -// -// int main() -// { -// // user_function's `T` will be deduced to `int` and `Ints...` -// // will be deduced to `0, 1, 2, 3, 4`. -// user_function(make_integer_sequence<int, 5>()); -// } -template <typename T, T... Ints> -struct integer_sequence { - using value_type = T; - static constexpr size_t size() noexcept { return sizeof...(Ints); } -}; - -// index_sequence -// -// A helper template for an `integer_sequence` of `size_t`, -// `absl::index_sequence` is designed to be a drop-in replacement for C++14's -// `std::index_sequence`. -template <size_t... Ints> -using index_sequence = integer_sequence<size_t, Ints...>; +// Historical note: Abseil once provided implementations of these +// abstractions for platforms that had not yet provided them. Those +// platforms are no longer supported. New code should simply use the +// the ones from std directly. +using std::index_sequence; +using std::index_sequence_for; +using std::integer_sequence; +using std::make_index_sequence; +using std::make_integer_sequence; namespace utility_internal { -template <typename Seq, size_t SeqSize, size_t Rem> -struct Extend; - -// Note that SeqSize == sizeof...(Ints). It's passed explicitly for efficiency. -template <typename T, T... Ints, size_t SeqSize> -struct Extend<integer_sequence<T, Ints...>, SeqSize, 0> { - using type = integer_sequence<T, Ints..., (Ints + SeqSize)...>; -}; - -template <typename T, T... Ints, size_t SeqSize> -struct Extend<integer_sequence<T, Ints...>, SeqSize, 1> { - using type = integer_sequence<T, Ints..., (Ints + SeqSize)..., 2 * SeqSize>; -}; - -// Recursion helper for 'make_integer_sequence<T, N>'. -// 'Gen<T, N>::type' is an alias for 'integer_sequence<T, 0, 1, ... N-1>'. -template <typename T, size_t N> -struct Gen { - using type = - typename Extend<typename Gen<T, N / 2>::type, N / 2, N % 2>::type; -}; - -template <typename T> -struct Gen<T, 0> { - using type = integer_sequence<T>; -}; - template <typename T> struct InPlaceTypeTag { explicit InPlaceTypeTag() = delete; @@ -131,32 +75,6 @@ struct InPlaceIndexTag { } // namespace utility_internal -// Compile-time sequences of integers - -// make_integer_sequence -// -// This template alias is equivalent to -// `integer_sequence<int, 0, 1, ..., N-1>`, and is designed to be a drop-in -// replacement for C++14's `std::make_integer_sequence`. -template <typename T, T N> -using make_integer_sequence = typename utility_internal::Gen<T, N>::type; - -// make_index_sequence -// -// This template alias is equivalent to `index_sequence<0, 1, ..., N-1>`, -// and is designed to be a drop-in replacement for C++14's -// `std::make_index_sequence`. -template <size_t N> -using make_index_sequence = make_integer_sequence<size_t, N>; - -// index_sequence_for -// -// Converts a typename pack into an index sequence of the same length, and -// is designed to be a drop-in replacement for C++14's -// `std::index_sequence_for()` -template <typename... Ts> -using index_sequence_for = make_index_sequence<sizeof...(Ts)>; - // Tag types #ifdef ABSL_USES_STD_OPTIONAL diff --git a/contrib/restricted/abseil-cpp/absl/utility/ya.make b/contrib/restricted/abseil-cpp/absl/utility/ya.make index 3b66e3e400..3f9af85fe6 100644 --- a/contrib/restricted/abseil-cpp/absl/utility/ya.make +++ b/contrib/restricted/abseil-cpp/absl/utility/ya.make @@ -6,9 +6,9 @@ LICENSE(Apache-2.0) LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(20230802.1) +VERSION(20240116.0) -ORIGINAL_SOURCE(https://github.com/abseil/abseil-cpp/archive/20230802.1.tar.gz) +ORIGINAL_SOURCE(https://github.com/abseil/abseil-cpp/archive/20240116.0.tar.gz) NO_RUNTIME() diff --git a/contrib/restricted/abseil-cpp/ya.make b/contrib/restricted/abseil-cpp/ya.make index c89fe4570a..36452ac10e 100644 --- a/contrib/restricted/abseil-cpp/ya.make +++ b/contrib/restricted/abseil-cpp/ya.make @@ -6,9 +6,9 @@ LICENSE(Apache-2.0) LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(20230802.1) +VERSION(20240116.0) -ORIGINAL_SOURCE(https://github.com/abseil/abseil-cpp/archive/20230802.1.tar.gz) +ORIGINAL_SOURCE(https://github.com/abseil/abseil-cpp/archive/20240116.0.tar.gz) PEERDIR( contrib/restricted/abseil-cpp/absl/algorithm |