diff options
author | anastasy888 <anastasy888@yandex-team.ru> | 2022-02-10 16:45:54 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:54 +0300 |
commit | 49f765d71da452ea93138a25559dfa68dd76c7f3 (patch) | |
tree | 1016041feb637349e401dcc0fa85217dd2c2c639 /contrib/restricted/abseil-cpp/absl/base/internal/spinlock.h | |
parent | 7353a3fdea9c67c256980c00a2b3b67f09b23a27 (diff) | |
download | ydb-49f765d71da452ea93138a25559dfa68dd76c7f3.tar.gz |
Restoring authorship annotation for <anastasy888@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'contrib/restricted/abseil-cpp/absl/base/internal/spinlock.h')
-rw-r--r-- | contrib/restricted/abseil-cpp/absl/base/internal/spinlock.h | 420 |
1 files changed, 210 insertions, 210 deletions
diff --git a/contrib/restricted/abseil-cpp/absl/base/internal/spinlock.h b/contrib/restricted/abseil-cpp/absl/base/internal/spinlock.h index ac40daff12..cafaaa786b 100644 --- a/contrib/restricted/abseil-cpp/absl/base/internal/spinlock.h +++ b/contrib/restricted/abseil-cpp/absl/base/internal/spinlock.h @@ -1,64 +1,64 @@ -// -// Copyright 2017 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -// Most users requiring mutual exclusion should use Mutex. +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Most users requiring mutual exclusion should use Mutex. // SpinLock is provided for use in two situations: // - for use by Abseil internal code that Mutex itself depends on -// - for async signal safety (see below) - -// SpinLock is async signal safe. If a spinlock is used within a signal -// handler, all code that acquires the lock must ensure that the signal cannot -// arrive while they are holding the lock. Typically, this is done by blocking -// the signal. +// - for async signal safety (see below) + +// SpinLock is async signal safe. If a spinlock is used within a signal +// handler, all code that acquires the lock must ensure that the signal cannot +// arrive while they are holding the lock. Typically, this is done by blocking +// the signal. // // Threads waiting on a SpinLock may be woken in an arbitrary order. - -#ifndef ABSL_BASE_INTERNAL_SPINLOCK_H_ -#define ABSL_BASE_INTERNAL_SPINLOCK_H_ - -#include <stdint.h> -#include <sys/types.h> - -#include <atomic> - -#include "absl/base/attributes.h" + +#ifndef ABSL_BASE_INTERNAL_SPINLOCK_H_ +#define ABSL_BASE_INTERNAL_SPINLOCK_H_ + +#include <stdint.h> +#include <sys/types.h> + +#include <atomic> + +#include "absl/base/attributes.h" #include "absl/base/const_init.h" -#include "absl/base/dynamic_annotations.h" -#include "absl/base/internal/low_level_scheduling.h" -#include "absl/base/internal/raw_logging.h" -#include "absl/base/internal/scheduling_mode.h" -#include "absl/base/internal/tsan_mutex_interface.h" -#include "absl/base/macros.h" -#include "absl/base/port.h" -#include "absl/base/thread_annotations.h" - -namespace absl { +#include "absl/base/dynamic_annotations.h" +#include "absl/base/internal/low_level_scheduling.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/scheduling_mode.h" +#include "absl/base/internal/tsan_mutex_interface.h" +#include "absl/base/macros.h" +#include "absl/base/port.h" +#include "absl/base/thread_annotations.h" + +namespace absl { ABSL_NAMESPACE_BEGIN -namespace base_internal { - -class ABSL_LOCKABLE SpinLock { - public: - SpinLock() : lockword_(kSpinLockCooperative) { - ABSL_TSAN_MUTEX_CREATE(this, __tsan_mutex_not_static); - } - - // Constructors that allow non-cooperative spinlocks to be created for use - // inside thread schedulers. Normal clients should not use these. - explicit SpinLock(base_internal::SchedulingMode mode); - +namespace base_internal { + +class ABSL_LOCKABLE SpinLock { + public: + SpinLock() : lockword_(kSpinLockCooperative) { + ABSL_TSAN_MUTEX_CREATE(this, __tsan_mutex_not_static); + } + + // Constructors that allow non-cooperative spinlocks to be created for use + // inside thread schedulers. Normal clients should not use these. + explicit SpinLock(base_internal::SchedulingMode mode); + // Constructor for global SpinLock instances. See absl/base/const_init.h. constexpr SpinLock(absl::ConstInitType, base_internal::SchedulingMode mode) : lockword_(IsCooperative(mode) ? kSpinLockCooperative : 0) {} @@ -67,81 +67,81 @@ class ABSL_LOCKABLE SpinLock { // Default but non-trivial destructor in some build configurations causes an // extra static initializer. #ifdef ABSL_INTERNAL_HAVE_TSAN_INTERFACE - ~SpinLock() { ABSL_TSAN_MUTEX_DESTROY(this, __tsan_mutex_not_static); } + ~SpinLock() { ABSL_TSAN_MUTEX_DESTROY(this, __tsan_mutex_not_static); } #else ~SpinLock() = default; #endif - - // Acquire this SpinLock. - inline void Lock() ABSL_EXCLUSIVE_LOCK_FUNCTION() { - ABSL_TSAN_MUTEX_PRE_LOCK(this, 0); - if (!TryLockImpl()) { - SlowLock(); - } - ABSL_TSAN_MUTEX_POST_LOCK(this, 0, 0); - } - - // Try to acquire this SpinLock without blocking and return true if the - // acquisition was successful. If the lock was not acquired, false is - // returned. If this SpinLock is free at the time of the call, TryLock - // will return true with high probability. - inline bool TryLock() ABSL_EXCLUSIVE_TRYLOCK_FUNCTION(true) { - ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_try_lock); - bool res = TryLockImpl(); - ABSL_TSAN_MUTEX_POST_LOCK( - this, __tsan_mutex_try_lock | (res ? 0 : __tsan_mutex_try_lock_failed), - 0); - return res; - } - - // Release this SpinLock, which must be held by the calling thread. - inline void Unlock() ABSL_UNLOCK_FUNCTION() { - ABSL_TSAN_MUTEX_PRE_UNLOCK(this, 0); - uint32_t lock_value = lockword_.load(std::memory_order_relaxed); - lock_value = lockword_.exchange(lock_value & kSpinLockCooperative, - std::memory_order_release); - - if ((lock_value & kSpinLockDisabledScheduling) != 0) { - base_internal::SchedulingGuard::EnableRescheduling(true); - } - if ((lock_value & kWaitTimeMask) != 0) { - // Collect contentionz profile info, and speed the wakeup of any waiter. - // The wait_cycles value indicates how long this thread spent waiting - // for the lock. - SlowUnlock(lock_value); - } - ABSL_TSAN_MUTEX_POST_UNLOCK(this, 0); - } - - // Determine if the lock is held. When the lock is held by the invoking - // thread, true will always be returned. Intended to be used as - // CHECK(lock.IsHeld()). - inline bool IsHeld() const { - return (lockword_.load(std::memory_order_relaxed) & kSpinLockHeld) != 0; - } - - protected: - // These should not be exported except for testing. - - // Store number of cycles between wait_start_time and wait_end_time in a - // lock value. - static uint32_t EncodeWaitCycles(int64_t wait_start_time, - int64_t wait_end_time); - - // Extract number of wait cycles in a lock value. - static uint64_t DecodeWaitCycles(uint32_t lock_value); - - // Provide access to protected method above. Use for testing only. - friend struct SpinLockTest; - - private: - // lockword_ is used to store the following: - // - // bit[0] encodes whether a lock is being held. - // bit[1] encodes whether a lock uses cooperative scheduling. + + // Acquire this SpinLock. + inline void Lock() ABSL_EXCLUSIVE_LOCK_FUNCTION() { + ABSL_TSAN_MUTEX_PRE_LOCK(this, 0); + if (!TryLockImpl()) { + SlowLock(); + } + ABSL_TSAN_MUTEX_POST_LOCK(this, 0, 0); + } + + // Try to acquire this SpinLock without blocking and return true if the + // acquisition was successful. If the lock was not acquired, false is + // returned. If this SpinLock is free at the time of the call, TryLock + // will return true with high probability. + inline bool TryLock() ABSL_EXCLUSIVE_TRYLOCK_FUNCTION(true) { + ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_try_lock); + bool res = TryLockImpl(); + ABSL_TSAN_MUTEX_POST_LOCK( + this, __tsan_mutex_try_lock | (res ? 0 : __tsan_mutex_try_lock_failed), + 0); + return res; + } + + // Release this SpinLock, which must be held by the calling thread. + inline void Unlock() ABSL_UNLOCK_FUNCTION() { + ABSL_TSAN_MUTEX_PRE_UNLOCK(this, 0); + uint32_t lock_value = lockword_.load(std::memory_order_relaxed); + lock_value = lockword_.exchange(lock_value & kSpinLockCooperative, + std::memory_order_release); + + if ((lock_value & kSpinLockDisabledScheduling) != 0) { + base_internal::SchedulingGuard::EnableRescheduling(true); + } + if ((lock_value & kWaitTimeMask) != 0) { + // Collect contentionz profile info, and speed the wakeup of any waiter. + // The wait_cycles value indicates how long this thread spent waiting + // for the lock. + SlowUnlock(lock_value); + } + ABSL_TSAN_MUTEX_POST_UNLOCK(this, 0); + } + + // Determine if the lock is held. When the lock is held by the invoking + // thread, true will always be returned. Intended to be used as + // CHECK(lock.IsHeld()). + inline bool IsHeld() const { + return (lockword_.load(std::memory_order_relaxed) & kSpinLockHeld) != 0; + } + + protected: + // These should not be exported except for testing. + + // Store number of cycles between wait_start_time and wait_end_time in a + // lock value. + static uint32_t EncodeWaitCycles(int64_t wait_start_time, + int64_t wait_end_time); + + // Extract number of wait cycles in a lock value. + static uint64_t DecodeWaitCycles(uint32_t lock_value); + + // Provide access to protected method above. Use for testing only. + friend struct SpinLockTest; + + private: + // lockword_ is used to store the following: + // + // bit[0] encodes whether a lock is being held. + // bit[1] encodes whether a lock uses cooperative scheduling. // bit[2] encodes whether the current lock holder disabled scheduling when // acquiring the lock. Only set when kSpinLockHeld is also set. - // bit[3:31] encodes time a lock spent on waiting as a 29-bit unsigned int. + // bit[3:31] encodes time a lock spent on waiting as a 29-bit unsigned int. // This is set by the lock holder to indicate how long it waited on // the lock before eventually acquiring it. The number of cycles is // encoded as a 29-bit unsigned int, or in the case that the current @@ -160,89 +160,89 @@ class ABSL_LOCKABLE SpinLock { // Includes kSpinLockSleeper. static constexpr uint32_t kWaitTimeMask = ~(kSpinLockHeld | kSpinLockCooperative | kSpinLockDisabledScheduling); - - // Returns true if the provided scheduling mode is cooperative. - static constexpr bool IsCooperative( - base_internal::SchedulingMode scheduling_mode) { - return scheduling_mode == base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL; - } - - uint32_t TryLockInternal(uint32_t lock_value, uint32_t wait_cycles); - void SlowLock() ABSL_ATTRIBUTE_COLD; - void SlowUnlock(uint32_t lock_value) ABSL_ATTRIBUTE_COLD; - uint32_t SpinLoop(); - - inline bool TryLockImpl() { - uint32_t lock_value = lockword_.load(std::memory_order_relaxed); - return (TryLockInternal(lock_value, 0) & kSpinLockHeld) == 0; - } - - std::atomic<uint32_t> lockword_; - - SpinLock(const SpinLock&) = delete; - SpinLock& operator=(const SpinLock&) = delete; -}; - -// Corresponding locker object that arranges to acquire a spinlock for -// the duration of a C++ scope. -class ABSL_SCOPED_LOCKABLE SpinLockHolder { - public: - inline explicit SpinLockHolder(SpinLock* l) ABSL_EXCLUSIVE_LOCK_FUNCTION(l) - : lock_(l) { - l->Lock(); - } - inline ~SpinLockHolder() ABSL_UNLOCK_FUNCTION() { lock_->Unlock(); } - - SpinLockHolder(const SpinLockHolder&) = delete; - SpinLockHolder& operator=(const SpinLockHolder&) = delete; - - private: - SpinLock* lock_; -}; - -// Register a hook for profiling support. -// -// The function pointer registered here will be called whenever a spinlock is -// contended. The callback is given an opaque handle to the contended spinlock -// and the number of wait cycles. This is thread-safe, but only a single -// profiler can be registered. It is an error to call this function multiple -// times with different arguments. -void RegisterSpinLockProfiler(void (*fn)(const void* lock, - int64_t wait_cycles)); - -//------------------------------------------------------------------------------ -// Public interface ends here. -//------------------------------------------------------------------------------ - -// If (result & kSpinLockHeld) == 0, then *this was successfully locked. -// Otherwise, returns last observed value for lockword_. -inline uint32_t SpinLock::TryLockInternal(uint32_t lock_value, - uint32_t wait_cycles) { - if ((lock_value & kSpinLockHeld) != 0) { - return lock_value; - } - - uint32_t sched_disabled_bit = 0; - if ((lock_value & kSpinLockCooperative) == 0) { - // For non-cooperative locks we must make sure we mark ourselves as - // non-reschedulable before we attempt to CompareAndSwap. - if (base_internal::SchedulingGuard::DisableRescheduling()) { - sched_disabled_bit = kSpinLockDisabledScheduling; - } - } - - if (!lockword_.compare_exchange_strong( - lock_value, - kSpinLockHeld | lock_value | wait_cycles | sched_disabled_bit, - std::memory_order_acquire, std::memory_order_relaxed)) { - base_internal::SchedulingGuard::EnableRescheduling(sched_disabled_bit != 0); - } - - return lock_value; -} - -} // namespace base_internal + + // Returns true if the provided scheduling mode is cooperative. + static constexpr bool IsCooperative( + base_internal::SchedulingMode scheduling_mode) { + return scheduling_mode == base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL; + } + + uint32_t TryLockInternal(uint32_t lock_value, uint32_t wait_cycles); + void SlowLock() ABSL_ATTRIBUTE_COLD; + void SlowUnlock(uint32_t lock_value) ABSL_ATTRIBUTE_COLD; + uint32_t SpinLoop(); + + inline bool TryLockImpl() { + uint32_t lock_value = lockword_.load(std::memory_order_relaxed); + return (TryLockInternal(lock_value, 0) & kSpinLockHeld) == 0; + } + + std::atomic<uint32_t> lockword_; + + SpinLock(const SpinLock&) = delete; + SpinLock& operator=(const SpinLock&) = delete; +}; + +// Corresponding locker object that arranges to acquire a spinlock for +// the duration of a C++ scope. +class ABSL_SCOPED_LOCKABLE SpinLockHolder { + public: + inline explicit SpinLockHolder(SpinLock* l) ABSL_EXCLUSIVE_LOCK_FUNCTION(l) + : lock_(l) { + l->Lock(); + } + inline ~SpinLockHolder() ABSL_UNLOCK_FUNCTION() { lock_->Unlock(); } + + SpinLockHolder(const SpinLockHolder&) = delete; + SpinLockHolder& operator=(const SpinLockHolder&) = delete; + + private: + SpinLock* lock_; +}; + +// Register a hook for profiling support. +// +// The function pointer registered here will be called whenever a spinlock is +// contended. The callback is given an opaque handle to the contended spinlock +// and the number of wait cycles. This is thread-safe, but only a single +// profiler can be registered. It is an error to call this function multiple +// times with different arguments. +void RegisterSpinLockProfiler(void (*fn)(const void* lock, + int64_t wait_cycles)); + +//------------------------------------------------------------------------------ +// Public interface ends here. +//------------------------------------------------------------------------------ + +// If (result & kSpinLockHeld) == 0, then *this was successfully locked. +// Otherwise, returns last observed value for lockword_. +inline uint32_t SpinLock::TryLockInternal(uint32_t lock_value, + uint32_t wait_cycles) { + if ((lock_value & kSpinLockHeld) != 0) { + return lock_value; + } + + uint32_t sched_disabled_bit = 0; + if ((lock_value & kSpinLockCooperative) == 0) { + // For non-cooperative locks we must make sure we mark ourselves as + // non-reschedulable before we attempt to CompareAndSwap. + if (base_internal::SchedulingGuard::DisableRescheduling()) { + sched_disabled_bit = kSpinLockDisabledScheduling; + } + } + + if (!lockword_.compare_exchange_strong( + lock_value, + kSpinLockHeld | lock_value | wait_cycles | sched_disabled_bit, + std::memory_order_acquire, std::memory_order_relaxed)) { + base_internal::SchedulingGuard::EnableRescheduling(sched_disabled_bit != 0); + } + + return lock_value; +} + +} // namespace base_internal ABSL_NAMESPACE_END -} // namespace absl - -#endif // ABSL_BASE_INTERNAL_SPINLOCK_H_ +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_SPINLOCK_H_ |