1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
|
#pragma once
#include <util/system/defaults.h>
#include <library/cpp/deprecated/atomic/atomic.h>
#include <util/system/spinlock.h>
#include <library/cpp/sse/sse.h> // The header chooses appropriate SSE support
static_assert(sizeof(TAtomic) == 8, "expect sizeof(TAtomic) == 8");
// we need explicit 32 bit operations to keep cache-line friendly packs
// so have to define some atomics additionaly to arcadia one
#ifdef _win_
#pragma intrinsic(_InterlockedCompareExchange)
#pragma intrinsic(_InterlockedExchangeAdd)
#pragma intrinsic(_InterlockedIncrement)
#pragma intrinsic(_InterlockedDecrement)
#endif
inline bool AtomicUi32Cas(volatile ui32* a, ui32 exchange, ui32 compare) {
#ifdef _win_
return _InterlockedCompareExchange((volatile long*)a, exchange, compare) == (long)compare;
#else
ui32 expected = compare;
return __atomic_compare_exchange_n(a, &expected, exchange, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
#endif
}
inline ui32 AtomicUi32Add(volatile ui32* a, ui32 add) {
#ifdef _win_
return _InterlockedExchangeAdd((volatile long*)a, add) + add;
#else
return __atomic_add_fetch(a, add, __ATOMIC_SEQ_CST);
#endif
}
inline ui32 AtomicUi32Sub(volatile ui32* a, ui32 sub) {
#ifdef _win_
return _InterlockedExchangeAdd((volatile long*)a, -(long)sub) - sub;
#else
return __atomic_sub_fetch(a, sub, __ATOMIC_SEQ_CST);
#endif
}
inline ui32 AtomicUi32Increment(volatile ui32* a) {
#ifdef _win_
return _InterlockedIncrement((volatile long*)a);
#else
return __atomic_add_fetch(a, 1, __ATOMIC_SEQ_CST);
#endif
}
inline ui32 AtomicUi32Decrement(volatile ui32* a) {
#ifdef _win_
return _InterlockedDecrement((volatile long*)a);
#else
return __atomic_sub_fetch(a, 1, __ATOMIC_SEQ_CST);
#endif
}
template <typename T>
inline void AtomicStore(volatile T* a, T x) {
static_assert(std::is_integral<T>::value || std::is_pointer<T>::value, "expect std::is_integral<T>::value || std::is_pointer<T>::value");
#ifdef _win_
*a = x;
#else
__atomic_store_n(a, x, __ATOMIC_RELEASE);
#endif
}
template <typename T>
inline void RelaxedStore(volatile T* a, T x) {
static_assert(std::is_integral<T>::value || std::is_pointer<T>::value, "expect std::is_integral<T>::value || std::is_pointer<T>::value");
#ifdef _win_
*a = x;
#else
__atomic_store_n(a, x, __ATOMIC_RELAXED);
#endif
}
template <typename T>
inline T AtomicLoad(volatile T* a) {
#ifdef _win_
return *a;
#else
return __atomic_load_n(a, __ATOMIC_ACQUIRE);
#endif
}
template <typename T>
inline T RelaxedLoad(volatile T* a) {
#ifdef _win_
return *a;
#else
return __atomic_load_n(a, __ATOMIC_RELAXED);
#endif
}
|
