aboutsummaryrefslogtreecommitdiffstats
path: root/contrib/restricted/abseil-cpp/absl/base/internal/spinlock.cc
diff options
context:
space:
mode:
authoranastasy888 <anastasy888@yandex-team.ru>2022-02-10 16:45:55 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:45:55 +0300
commit3a7a498715ef1b66f5054455421b845e45e3a653 (patch)
tree1a2c5ffcf89eb53ecd79dbc9bc0a195c27404d0c /contrib/restricted/abseil-cpp/absl/base/internal/spinlock.cc
parent49f765d71da452ea93138a25559dfa68dd76c7f3 (diff)
downloadydb-3a7a498715ef1b66f5054455421b845e45e3a653.tar.gz
Restoring authorship annotation for <anastasy888@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'contrib/restricted/abseil-cpp/absl/base/internal/spinlock.cc')
-rw-r--r--contrib/restricted/abseil-cpp/absl/base/internal/spinlock.cc378
1 files changed, 189 insertions, 189 deletions
diff --git a/contrib/restricted/abseil-cpp/absl/base/internal/spinlock.cc b/contrib/restricted/abseil-cpp/absl/base/internal/spinlock.cc
index a3b39f2c28..35c0696a34 100644
--- a/contrib/restricted/abseil-cpp/absl/base/internal/spinlock.cc
+++ b/contrib/restricted/abseil-cpp/absl/base/internal/spinlock.cc
@@ -1,71 +1,71 @@
-// Copyright 2017 The Abseil Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "absl/base/internal/spinlock.h"
-
-#include <algorithm>
-#include <atomic>
-#include <limits>
-
-#include "absl/base/attributes.h"
-#include "absl/base/internal/atomic_hook.h"
-#include "absl/base/internal/cycleclock.h"
-#include "absl/base/internal/spinlock_wait.h"
-#include "absl/base/internal/sysinfo.h" /* For NumCPUs() */
-#include "absl/base/call_once.h"
-
-// Description of lock-word:
-// 31..00: [............................3][2][1][0]
-//
-// [0]: kSpinLockHeld
-// [1]: kSpinLockCooperative
-// [2]: kSpinLockDisabledScheduling
-// [31..3]: ONLY kSpinLockSleeper OR
-// Wait time in cycles >> PROFILE_TIMESTAMP_SHIFT
-//
-// Detailed descriptions:
-//
-// Bit [0]: The lock is considered held iff kSpinLockHeld is set.
-//
-// Bit [1]: Eligible waiters (e.g. Fibers) may co-operatively reschedule when
-// contended iff kSpinLockCooperative is set.
-//
-// Bit [2]: This bit is exclusive from bit [1]. It is used only by a
-// non-cooperative lock. When set, indicates that scheduling was
-// successfully disabled when the lock was acquired. May be unset,
-// even if non-cooperative, if a ThreadIdentity did not yet exist at
-// time of acquisition.
-//
-// Bit [3]: If this is the only upper bit ([31..3]) set then this lock was
-// acquired without contention, however, at least one waiter exists.
-//
-// Otherwise, bits [31..3] represent the time spent by the current lock
-// holder to acquire the lock. There may be outstanding waiter(s).
-
-namespace absl {
+// Copyright 2017 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/base/internal/spinlock.h"
+
+#include <algorithm>
+#include <atomic>
+#include <limits>
+
+#include "absl/base/attributes.h"
+#include "absl/base/internal/atomic_hook.h"
+#include "absl/base/internal/cycleclock.h"
+#include "absl/base/internal/spinlock_wait.h"
+#include "absl/base/internal/sysinfo.h" /* For NumCPUs() */
+#include "absl/base/call_once.h"
+
+// Description of lock-word:
+// 31..00: [............................3][2][1][0]
+//
+// [0]: kSpinLockHeld
+// [1]: kSpinLockCooperative
+// [2]: kSpinLockDisabledScheduling
+// [31..3]: ONLY kSpinLockSleeper OR
+// Wait time in cycles >> PROFILE_TIMESTAMP_SHIFT
+//
+// Detailed descriptions:
+//
+// Bit [0]: The lock is considered held iff kSpinLockHeld is set.
+//
+// Bit [1]: Eligible waiters (e.g. Fibers) may co-operatively reschedule when
+// contended iff kSpinLockCooperative is set.
+//
+// Bit [2]: This bit is exclusive from bit [1]. It is used only by a
+// non-cooperative lock. When set, indicates that scheduling was
+// successfully disabled when the lock was acquired. May be unset,
+// even if non-cooperative, if a ThreadIdentity did not yet exist at
+// time of acquisition.
+//
+// Bit [3]: If this is the only upper bit ([31..3]) set then this lock was
+// acquired without contention, however, at least one waiter exists.
+//
+// Otherwise, bits [31..3] represent the time spent by the current lock
+// holder to acquire the lock. There may be outstanding waiter(s).
+
+namespace absl {
ABSL_NAMESPACE_BEGIN
-namespace base_internal {
-
+namespace base_internal {
+
ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static base_internal::AtomicHook<void (*)(
const void *lock, int64_t wait_cycles)>
- submit_profile_data;
-
-void RegisterSpinLockProfiler(void (*fn)(const void *contendedlock,
- int64_t wait_cycles)) {
- submit_profile_data.Store(fn);
-}
-
+ submit_profile_data;
+
+void RegisterSpinLockProfiler(void (*fn)(const void *contendedlock,
+ int64_t wait_cycles)) {
+ submit_profile_data.Store(fn);
+}
+
// Static member variable definitions.
constexpr uint32_t SpinLock::kSpinLockHeld;
constexpr uint32_t SpinLock::kSpinLockCooperative;
@@ -73,38 +73,38 @@ constexpr uint32_t SpinLock::kSpinLockDisabledScheduling;
constexpr uint32_t SpinLock::kSpinLockSleeper;
constexpr uint32_t SpinLock::kWaitTimeMask;
-// Uncommon constructors.
-SpinLock::SpinLock(base_internal::SchedulingMode mode)
- : lockword_(IsCooperative(mode) ? kSpinLockCooperative : 0) {
- ABSL_TSAN_MUTEX_CREATE(this, __tsan_mutex_not_static);
-}
-
-// Monitor the lock to see if its value changes within some time period
-// (adaptive_spin_count loop iterations). The last value read from the lock
-// is returned from the method.
-uint32_t SpinLock::SpinLoop() {
- // We are already in the slow path of SpinLock, initialize the
- // adaptive_spin_count here.
- ABSL_CONST_INIT static absl::once_flag init_adaptive_spin_count;
- ABSL_CONST_INIT static int adaptive_spin_count = 0;
- base_internal::LowLevelCallOnce(&init_adaptive_spin_count, []() {
- adaptive_spin_count = base_internal::NumCPUs() > 1 ? 1000 : 1;
- });
-
- int c = adaptive_spin_count;
- uint32_t lock_value;
- do {
- lock_value = lockword_.load(std::memory_order_relaxed);
- } while ((lock_value & kSpinLockHeld) != 0 && --c > 0);
- return lock_value;
-}
-
-void SpinLock::SlowLock() {
- uint32_t lock_value = SpinLoop();
- lock_value = TryLockInternal(lock_value, 0);
- if ((lock_value & kSpinLockHeld) == 0) {
- return;
- }
+// Uncommon constructors.
+SpinLock::SpinLock(base_internal::SchedulingMode mode)
+ : lockword_(IsCooperative(mode) ? kSpinLockCooperative : 0) {
+ ABSL_TSAN_MUTEX_CREATE(this, __tsan_mutex_not_static);
+}
+
+// Monitor the lock to see if its value changes within some time period
+// (adaptive_spin_count loop iterations). The last value read from the lock
+// is returned from the method.
+uint32_t SpinLock::SpinLoop() {
+ // We are already in the slow path of SpinLock, initialize the
+ // adaptive_spin_count here.
+ ABSL_CONST_INIT static absl::once_flag init_adaptive_spin_count;
+ ABSL_CONST_INIT static int adaptive_spin_count = 0;
+ base_internal::LowLevelCallOnce(&init_adaptive_spin_count, []() {
+ adaptive_spin_count = base_internal::NumCPUs() > 1 ? 1000 : 1;
+ });
+
+ int c = adaptive_spin_count;
+ uint32_t lock_value;
+ do {
+ lock_value = lockword_.load(std::memory_order_relaxed);
+ } while ((lock_value & kSpinLockHeld) != 0 && --c > 0);
+ return lock_value;
+}
+
+void SpinLock::SlowLock() {
+ uint32_t lock_value = SpinLoop();
+ lock_value = TryLockInternal(lock_value, 0);
+ if ((lock_value & kSpinLockHeld) == 0) {
+ return;
+ }
base_internal::SchedulingMode scheduling_mode;
if ((lock_value & kSpinLockCooperative) != 0) {
@@ -113,34 +113,34 @@ void SpinLock::SlowLock() {
scheduling_mode = base_internal::SCHEDULE_KERNEL_ONLY;
}
- // The lock was not obtained initially, so this thread needs to wait for
- // it. Record the current timestamp in the local variable wait_start_time
- // so the total wait time can be stored in the lockword once this thread
- // obtains the lock.
- int64_t wait_start_time = CycleClock::Now();
- uint32_t wait_cycles = 0;
- int lock_wait_call_count = 0;
- while ((lock_value & kSpinLockHeld) != 0) {
- // If the lock is currently held, but not marked as having a sleeper, mark
- // it as having a sleeper.
- if ((lock_value & kWaitTimeMask) == 0) {
- // Here, just "mark" that the thread is going to sleep. Don't store the
+ // The lock was not obtained initially, so this thread needs to wait for
+ // it. Record the current timestamp in the local variable wait_start_time
+ // so the total wait time can be stored in the lockword once this thread
+ // obtains the lock.
+ int64_t wait_start_time = CycleClock::Now();
+ uint32_t wait_cycles = 0;
+ int lock_wait_call_count = 0;
+ while ((lock_value & kSpinLockHeld) != 0) {
+ // If the lock is currently held, but not marked as having a sleeper, mark
+ // it as having a sleeper.
+ if ((lock_value & kWaitTimeMask) == 0) {
+ // Here, just "mark" that the thread is going to sleep. Don't store the
// lock wait time in the lock -- the lock word stores the amount of time
// that the current holder waited before acquiring the lock, not the wait
// time of any thread currently waiting to acquire it.
- if (lockword_.compare_exchange_strong(
- lock_value, lock_value | kSpinLockSleeper,
- std::memory_order_relaxed, std::memory_order_relaxed)) {
- // Successfully transitioned to kSpinLockSleeper. Pass
- // kSpinLockSleeper to the SpinLockWait routine to properly indicate
- // the last lock_value observed.
- lock_value |= kSpinLockSleeper;
- } else if ((lock_value & kSpinLockHeld) == 0) {
- // Lock is free again, so try and acquire it before sleeping. The
- // new lock state will be the number of cycles this thread waited if
- // this thread obtains the lock.
- lock_value = TryLockInternal(lock_value, wait_cycles);
- continue; // Skip the delay at the end of the loop.
+ if (lockword_.compare_exchange_strong(
+ lock_value, lock_value | kSpinLockSleeper,
+ std::memory_order_relaxed, std::memory_order_relaxed)) {
+ // Successfully transitioned to kSpinLockSleeper. Pass
+ // kSpinLockSleeper to the SpinLockWait routine to properly indicate
+ // the last lock_value observed.
+ lock_value |= kSpinLockSleeper;
+ } else if ((lock_value & kSpinLockHeld) == 0) {
+ // Lock is free again, so try and acquire it before sleeping. The
+ // new lock state will be the number of cycles this thread waited if
+ // this thread obtains the lock.
+ lock_value = TryLockInternal(lock_value, wait_cycles);
+ continue; // Skip the delay at the end of the loop.
} else if ((lock_value & kWaitTimeMask) == 0) {
// The lock is still held, without a waiter being marked, but something
// else about the lock word changed, causing our CAS to fail. For
@@ -149,81 +149,81 @@ void SpinLock::SlowLock() {
// set that flag. In this case, attempt again to mark ourselves as a
// waiter.
continue;
- }
- }
-
- // SpinLockDelay() calls into fiber scheduler, we need to see
- // synchronization there to avoid false positives.
- ABSL_TSAN_MUTEX_PRE_DIVERT(this, 0);
- // Wait for an OS specific delay.
- base_internal::SpinLockDelay(&lockword_, lock_value, ++lock_wait_call_count,
- scheduling_mode);
- ABSL_TSAN_MUTEX_POST_DIVERT(this, 0);
- // Spin again after returning from the wait routine to give this thread
- // some chance of obtaining the lock.
- lock_value = SpinLoop();
- wait_cycles = EncodeWaitCycles(wait_start_time, CycleClock::Now());
- lock_value = TryLockInternal(lock_value, wait_cycles);
- }
-}
-
-void SpinLock::SlowUnlock(uint32_t lock_value) {
- base_internal::SpinLockWake(&lockword_,
- false); // wake waiter if necessary
-
- // If our acquisition was contended, collect contentionz profile info. We
- // reserve a unitary wait time to represent that a waiter exists without our
- // own acquisition having been contended.
- if ((lock_value & kWaitTimeMask) != kSpinLockSleeper) {
- const uint64_t wait_cycles = DecodeWaitCycles(lock_value);
- ABSL_TSAN_MUTEX_PRE_DIVERT(this, 0);
- submit_profile_data(this, wait_cycles);
- ABSL_TSAN_MUTEX_POST_DIVERT(this, 0);
- }
-}
-
-// We use the upper 29 bits of the lock word to store the time spent waiting to
-// acquire this lock. This is reported by contentionz profiling. Since the
-// lower bits of the cycle counter wrap very quickly on high-frequency
+ }
+ }
+
+ // SpinLockDelay() calls into fiber scheduler, we need to see
+ // synchronization there to avoid false positives.
+ ABSL_TSAN_MUTEX_PRE_DIVERT(this, 0);
+ // Wait for an OS specific delay.
+ base_internal::SpinLockDelay(&lockword_, lock_value, ++lock_wait_call_count,
+ scheduling_mode);
+ ABSL_TSAN_MUTEX_POST_DIVERT(this, 0);
+ // Spin again after returning from the wait routine to give this thread
+ // some chance of obtaining the lock.
+ lock_value = SpinLoop();
+ wait_cycles = EncodeWaitCycles(wait_start_time, CycleClock::Now());
+ lock_value = TryLockInternal(lock_value, wait_cycles);
+ }
+}
+
+void SpinLock::SlowUnlock(uint32_t lock_value) {
+ base_internal::SpinLockWake(&lockword_,
+ false); // wake waiter if necessary
+
+ // If our acquisition was contended, collect contentionz profile info. We
+ // reserve a unitary wait time to represent that a waiter exists without our
+ // own acquisition having been contended.
+ if ((lock_value & kWaitTimeMask) != kSpinLockSleeper) {
+ const uint64_t wait_cycles = DecodeWaitCycles(lock_value);
+ ABSL_TSAN_MUTEX_PRE_DIVERT(this, 0);
+ submit_profile_data(this, wait_cycles);
+ ABSL_TSAN_MUTEX_POST_DIVERT(this, 0);
+ }
+}
+
+// We use the upper 29 bits of the lock word to store the time spent waiting to
+// acquire this lock. This is reported by contentionz profiling. Since the
+// lower bits of the cycle counter wrap very quickly on high-frequency
// processors we divide to reduce the granularity to 2^kProfileTimestampShift
-// sized units. On a 4Ghz machine this will lose track of wait times greater
-// than (2^29/4 Ghz)*128 =~ 17.2 seconds. Such waits should be extremely rare.
+// sized units. On a 4Ghz machine this will lose track of wait times greater
+// than (2^29/4 Ghz)*128 =~ 17.2 seconds. Such waits should be extremely rare.
static constexpr int kProfileTimestampShift = 7;
-
+
// We currently reserve the lower 3 bits.
static constexpr int kLockwordReservedShift = 3;
-uint32_t SpinLock::EncodeWaitCycles(int64_t wait_start_time,
- int64_t wait_end_time) {
- static const int64_t kMaxWaitTime =
+uint32_t SpinLock::EncodeWaitCycles(int64_t wait_start_time,
+ int64_t wait_end_time) {
+ static const int64_t kMaxWaitTime =
std::numeric_limits<uint32_t>::max() >> kLockwordReservedShift;
- int64_t scaled_wait_time =
+ int64_t scaled_wait_time =
(wait_end_time - wait_start_time) >> kProfileTimestampShift;
-
- // Return a representation of the time spent waiting that can be stored in
- // the lock word's upper bits.
- uint32_t clamped = static_cast<uint32_t>(
+
+ // Return a representation of the time spent waiting that can be stored in
+ // the lock word's upper bits.
+ uint32_t clamped = static_cast<uint32_t>(
std::min(scaled_wait_time, kMaxWaitTime) << kLockwordReservedShift);
-
- if (clamped == 0) {
- return kSpinLockSleeper; // Just wake waiters, but don't record contention.
- }
- // Bump up value if necessary to avoid returning kSpinLockSleeper.
- const uint32_t kMinWaitTime =
+
+ if (clamped == 0) {
+ return kSpinLockSleeper; // Just wake waiters, but don't record contention.
+ }
+ // Bump up value if necessary to avoid returning kSpinLockSleeper.
+ const uint32_t kMinWaitTime =
kSpinLockSleeper + (1 << kLockwordReservedShift);
- if (clamped == kSpinLockSleeper) {
- return kMinWaitTime;
- }
- return clamped;
-}
-
-uint64_t SpinLock::DecodeWaitCycles(uint32_t lock_value) {
- // Cast to uint32_t first to ensure bits [63:32] are cleared.
- const uint64_t scaled_wait_time =
- static_cast<uint32_t>(lock_value & kWaitTimeMask);
+ if (clamped == kSpinLockSleeper) {
+ return kMinWaitTime;
+ }
+ return clamped;
+}
+
+uint64_t SpinLock::DecodeWaitCycles(uint32_t lock_value) {
+ // Cast to uint32_t first to ensure bits [63:32] are cleared.
+ const uint64_t scaled_wait_time =
+ static_cast<uint32_t>(lock_value & kWaitTimeMask);
return scaled_wait_time << (kProfileTimestampShift - kLockwordReservedShift);
-}
-
-} // namespace base_internal
+}
+
+} // namespace base_internal
ABSL_NAMESPACE_END
-} // namespace absl
+} // namespace absl