diff options
author | mikhnenko <mikhnenko@yandex-team.com> | 2023-09-20 11:01:13 +0300 |
---|---|---|
committer | mikhnenko <mikhnenko@yandex-team.com> | 2023-09-20 11:29:08 +0300 |
commit | f5b051f5e09f00265229985ba5fd47aa5c8959ff (patch) | |
tree | 7e11605ad566f09451623a1ac34cc644bae804ab /contrib/libs/cxxsupp/libcxxmsvc/src/atomic.cpp | |
parent | 8ed995cdfb22a3dfa790b0a6ae7750d9dc85db7b (diff) | |
download | ydb-f5b051f5e09f00265229985ba5fd47aa5c8959ff.tar.gz |
Split libcxx on msvc/other
Diffstat (limited to 'contrib/libs/cxxsupp/libcxxmsvc/src/atomic.cpp')
-rw-r--r-- | contrib/libs/cxxsupp/libcxxmsvc/src/atomic.cpp | 194 |
1 files changed, 194 insertions, 0 deletions
diff --git a/contrib/libs/cxxsupp/libcxxmsvc/src/atomic.cpp b/contrib/libs/cxxsupp/libcxxmsvc/src/atomic.cpp new file mode 100644 index 0000000000..5968aafb53 --- /dev/null +++ b/contrib/libs/cxxsupp/libcxxmsvc/src/atomic.cpp @@ -0,0 +1,194 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <__config> +#ifndef _LIBCPP_HAS_NO_THREADS + +#include <atomic> +#include <climits> +#include <functional> +#include <thread> + +#ifdef __linux__ + +#include <unistd.h> +#include <linux/futex.h> +#include <sys/syscall.h> + +// libc++ uses SYS_futex as a universal syscall name. However, on 32 bit architectures +// with a 64 bit time_t, we need to specify SYS_futex_time64. +#if !defined(SYS_futex) && defined(SYS_futex_time64) +# define SYS_futex SYS_futex_time64 +#endif + +#else // <- Add other operating systems here + +// Baseline needs no new headers + +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +#ifdef __linux__ + +static void __libcpp_platform_wait_on_address(__cxx_atomic_contention_t const volatile* __ptr, + __cxx_contention_t __val) +{ + static constexpr timespec __timeout = { 2, 0 }; + syscall(SYS_futex, __ptr, FUTEX_WAIT_PRIVATE, __val, &__timeout, 0, 0); +} + +static void __libcpp_platform_wake_by_address(__cxx_atomic_contention_t const volatile* __ptr, + bool __notify_one) +{ + syscall(SYS_futex, __ptr, FUTEX_WAKE_PRIVATE, __notify_one ? 1 : INT_MAX, 0, 0, 0); +} + +#elif defined(__APPLE__) && defined(_LIBCPP_USE_ULOCK) + +extern "C" int __ulock_wait(uint32_t operation, void *addr, uint64_t value, + uint32_t timeout); /* timeout is specified in microseconds */ +extern "C" int __ulock_wake(uint32_t operation, void *addr, uint64_t wake_value); + +#define UL_COMPARE_AND_WAIT 1 +#define ULF_WAKE_ALL 0x00000100 + +static void __libcpp_platform_wait_on_address(__cxx_atomic_contention_t const volatile* __ptr, + __cxx_contention_t __val) +{ + __ulock_wait(UL_COMPARE_AND_WAIT, + const_cast<__cxx_atomic_contention_t*>(__ptr), __val, 0); +} + +static void __libcpp_platform_wake_by_address(__cxx_atomic_contention_t const volatile* __ptr, + bool __notify_one) +{ + __ulock_wake(UL_COMPARE_AND_WAIT | (__notify_one ? 0 : ULF_WAKE_ALL), + const_cast<__cxx_atomic_contention_t*>(__ptr), 0); +} + +#else // <- Add other operating systems here + +// Baseline is just a timed backoff + +static void __libcpp_platform_wait_on_address(__cxx_atomic_contention_t const volatile* __ptr, + __cxx_contention_t __val) +{ + __libcpp_thread_poll_with_backoff([=]() -> bool { + return !__cxx_nonatomic_compare_equal(__cxx_atomic_load(__ptr, memory_order_relaxed), __val); + }, __libcpp_timed_backoff_policy()); +} + +static void __libcpp_platform_wake_by_address(__cxx_atomic_contention_t const volatile*, bool) { } + +#endif // __linux__ + +static constexpr size_t __libcpp_contention_table_size = (1 << 8); /* < there's no magic in this number */ + +struct alignas(64) /* aim to avoid false sharing */ __libcpp_contention_table_entry +{ + __cxx_atomic_contention_t __contention_state; + __cxx_atomic_contention_t __platform_state; + inline constexpr __libcpp_contention_table_entry() : + __contention_state(0), __platform_state(0) { } +}; + +static __libcpp_contention_table_entry __libcpp_contention_table[ __libcpp_contention_table_size ]; + +static hash<void const volatile*> __libcpp_contention_hasher; + +static __libcpp_contention_table_entry* __libcpp_contention_state(void const volatile * p) +{ + return &__libcpp_contention_table[__libcpp_contention_hasher(p) & (__libcpp_contention_table_size - 1)]; +} + +/* Given an atomic to track contention and an atomic to actually wait on, which may be + the same atomic, we try to detect contention to avoid spuriously calling the platform. */ + +static void __libcpp_contention_notify(__cxx_atomic_contention_t volatile* __contention_state, + __cxx_atomic_contention_t const volatile* __platform_state, + bool __notify_one) +{ + if(0 != __cxx_atomic_load(__contention_state, memory_order_seq_cst)) + // We only call 'wake' if we consumed a contention bit here. + __libcpp_platform_wake_by_address(__platform_state, __notify_one); +} +static __cxx_contention_t __libcpp_contention_monitor_for_wait(__cxx_atomic_contention_t volatile* __contention_state, + __cxx_atomic_contention_t const volatile* __platform_state) +{ + // We will monitor this value. + return __cxx_atomic_load(__platform_state, memory_order_acquire); +} +static void __libcpp_contention_wait(__cxx_atomic_contention_t volatile* __contention_state, + __cxx_atomic_contention_t const volatile* __platform_state, + __cxx_contention_t __old_value) +{ + __cxx_atomic_fetch_add(__contention_state, __cxx_contention_t(1), memory_order_seq_cst); + // We sleep as long as the monitored value hasn't changed. + __libcpp_platform_wait_on_address(__platform_state, __old_value); + __cxx_atomic_fetch_sub(__contention_state, __cxx_contention_t(1), memory_order_release); +} + +/* When the incoming atomic is the wrong size for the platform wait size, need to + launder the value sequence through an atomic from our table. */ + +static void __libcpp_atomic_notify(void const volatile* __location) +{ + auto const __entry = __libcpp_contention_state(__location); + // The value sequence laundering happens on the next line below. + __cxx_atomic_fetch_add(&__entry->__platform_state, __cxx_contention_t(1), memory_order_release); + __libcpp_contention_notify(&__entry->__contention_state, + &__entry->__platform_state, + false /* when laundering, we can't handle notify_one */); +} +_LIBCPP_EXPORTED_FROM_ABI +void __cxx_atomic_notify_one(void const volatile* __location) + { __libcpp_atomic_notify(__location); } +_LIBCPP_EXPORTED_FROM_ABI +void __cxx_atomic_notify_all(void const volatile* __location) + { __libcpp_atomic_notify(__location); } +_LIBCPP_EXPORTED_FROM_ABI +__cxx_contention_t __libcpp_atomic_monitor(void const volatile* __location) +{ + auto const __entry = __libcpp_contention_state(__location); + return __libcpp_contention_monitor_for_wait(&__entry->__contention_state, &__entry->__platform_state); +} +_LIBCPP_EXPORTED_FROM_ABI +void __libcpp_atomic_wait(void const volatile* __location, __cxx_contention_t __old_value) +{ + auto const __entry = __libcpp_contention_state(__location); + __libcpp_contention_wait(&__entry->__contention_state, &__entry->__platform_state, __old_value); +} + +/* When the incoming atomic happens to be the platform wait size, we still need to use the + table for the contention detection, but we can use the atomic directly for the wait. */ + +_LIBCPP_EXPORTED_FROM_ABI +void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile* __location) +{ + __libcpp_contention_notify(&__libcpp_contention_state(__location)->__contention_state, __location, true); +} +_LIBCPP_EXPORTED_FROM_ABI +void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile* __location) +{ + __libcpp_contention_notify(&__libcpp_contention_state(__location)->__contention_state, __location, false); +} +_LIBCPP_EXPORTED_FROM_ABI +__cxx_contention_t __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile* __location) +{ + return __libcpp_contention_monitor_for_wait(&__libcpp_contention_state(__location)->__contention_state, __location); +} +_LIBCPP_EXPORTED_FROM_ABI +void __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile* __location, __cxx_contention_t __old_value) +{ + __libcpp_contention_wait(&__libcpp_contention_state(__location)->__contention_state, __location, __old_value); +} + +_LIBCPP_END_NAMESPACE_STD + +#endif //_LIBCPP_HAS_NO_THREADS |