1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
|
//
// Copyright 2020 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef ABSL_FLAGS_INTERNAL_SEQUENCE_LOCK_H_
#define ABSL_FLAGS_INTERNAL_SEQUENCE_LOCK_H_
#include <stddef.h>
#include <stdint.h>
#include <atomic>
#include <cassert>
#include <cstring>
#include "absl/base/optimization.h"
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace flags_internal {
// Align 'x' up to the nearest 'align' bytes.
inline constexpr size_t AlignUp(size_t x, size_t align) {
return align * ((x + align - 1) / align);
}
// A SequenceLock implements lock-free reads. A sequence counter is incremented
// before and after each write, and readers access the counter before and after
// accessing the protected data. If the counter is verified to not change during
// the access, and the sequence counter value was even, then the reader knows
// that the read was race-free and valid. Otherwise, the reader must fall back
// to a Mutex-based code path.
//
// This particular SequenceLock starts in an "uninitialized" state in which
// TryRead() returns false. It must be enabled by calling MarkInitialized().
// This serves as a marker that the associated flag value has not yet been
// initialized and a slow path needs to be taken.
//
// The memory reads and writes protected by this lock must use the provided
// `TryRead()` and `Write()` functions. These functions behave similarly to
// `memcpy()`, with one oddity: the protected data must be an array of
// `std::atomic<uint64>`. This is to comply with the C++ standard, which
// considers data races on non-atomic objects to be undefined behavior. See "Can
// Seqlocks Get Along With Programming Language Memory Models?"[1] by Hans J.
// Boehm for more details.
//
// [1] https://www.hpl.hp.com/techreports/2012/HPL-2012-68.pdf
class SequenceLock {
public:
constexpr SequenceLock() : lock_(kUninitialized) {}
// Mark that this lock is ready for use.
void MarkInitialized() {
assert(lock_.load(std::memory_order_relaxed) == kUninitialized);
lock_.store(0, std::memory_order_release);
}
// Copy "size" bytes of data from "src" to "dst", protected as a read-side
// critical section of the sequence lock.
//
// Unlike traditional sequence lock implementations which loop until getting a
// clean read, this implementation returns false in the case of concurrent
// calls to `Write`. In such a case, the caller should fall back to a
// locking-based slow path.
//
// Returns false if the sequence lock was not yet marked as initialized.
//
// NOTE: If this returns false, "dst" may be overwritten with undefined
// (potentially uninitialized) data.
bool TryRead(void* dst, const std::atomic<uint64_t>* src, size_t size) const {
// Acquire barrier ensures that no loads done by f() are reordered
// above the first load of the sequence counter.
int64_t seq_before = lock_.load(std::memory_order_acquire);
if (ABSL_PREDICT_FALSE(seq_before & 1) == 1) return false;
RelaxedCopyFromAtomic(dst, src, size);
// Another acquire fence ensures that the load of 'lock_' below is
// strictly ordered after the RelaxedCopyToAtomic call above.
std::atomic_thread_fence(std::memory_order_acquire);
int64_t seq_after = lock_.load(std::memory_order_relaxed);
return ABSL_PREDICT_TRUE(seq_before == seq_after);
}
// Copy "size" bytes from "src" to "dst" as a write-side critical section
// of the sequence lock. Any concurrent readers will be forced to retry
// until they get a read that does not conflict with this write.
//
// This call must be externally synchronized against other calls to Write,
// but may proceed concurrently with reads.
void Write(std::atomic<uint64_t>* dst, const void* src, size_t size) {
// We can use relaxed instructions to increment the counter since we
// are extenally synchronized. The std::atomic_thread_fence below
// ensures that the counter updates don't get interleaved with the
// copy to the data.
int64_t orig_seq = lock_.load(std::memory_order_relaxed);
assert((orig_seq & 1) == 0); // Must be initially unlocked.
lock_.store(orig_seq + 1, std::memory_order_relaxed);
// We put a release fence between update to lock_ and writes to shared data.
// Thus all stores to shared data are effectively release operations and
// update to lock_ above cannot be re-ordered past any of them. Note that
// this barrier is not for the fetch_add above. A release barrier for the
// fetch_add would be before it, not after.
std::atomic_thread_fence(std::memory_order_release);
RelaxedCopyToAtomic(dst, src, size);
// "Release" semantics ensure that none of the writes done by
// RelaxedCopyToAtomic() can be reordered after the following modification.
lock_.store(orig_seq + 2, std::memory_order_release);
}
// Return the number of times that Write() has been called.
//
// REQUIRES: This must be externally synchronized against concurrent calls to
// `Write()` or `IncrementModificationCount()`.
// REQUIRES: `MarkInitialized()` must have been previously called.
int64_t ModificationCount() const {
int64_t val = lock_.load(std::memory_order_relaxed);
assert(val != kUninitialized && (val & 1) == 0);
return val / 2;
}
// REQUIRES: This must be externally synchronized against concurrent calls to
// `Write()` or `ModificationCount()`.
// REQUIRES: `MarkInitialized()` must have been previously called.
void IncrementModificationCount() {
int64_t val = lock_.load(std::memory_order_relaxed);
assert(val != kUninitialized);
lock_.store(val + 2, std::memory_order_relaxed);
}
private:
// Perform the equivalent of "memcpy(dst, src, size)", but using relaxed
// atomics.
static void RelaxedCopyFromAtomic(void* dst, const std::atomic<uint64_t>* src,
size_t size) {
char* dst_byte = static_cast<char*>(dst);
while (size >= sizeof(uint64_t)) {
uint64_t word = src->load(std::memory_order_relaxed);
std::memcpy(dst_byte, &word, sizeof(word));
dst_byte += sizeof(word);
src++;
size -= sizeof(word);
}
if (size > 0) {
uint64_t word = src->load(std::memory_order_relaxed);
std::memcpy(dst_byte, &word, size);
}
}
// Perform the equivalent of "memcpy(dst, src, size)", but using relaxed
// atomics.
static void RelaxedCopyToAtomic(std::atomic<uint64_t>* dst, const void* src,
size_t size) {
const char* src_byte = static_cast<const char*>(src);
while (size >= sizeof(uint64_t)) {
uint64_t word;
std::memcpy(&word, src_byte, sizeof(word));
dst->store(word, std::memory_order_relaxed);
src_byte += sizeof(word);
dst++;
size -= sizeof(word);
}
if (size > 0) {
uint64_t word = 0;
std::memcpy(&word, src_byte, size);
dst->store(word, std::memory_order_relaxed);
}
}
static constexpr int64_t kUninitialized = -1;
std::atomic<int64_t> lock_;
};
} // namespace flags_internal
ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_FLAGS_INTERNAL_SEQUENCE_LOCK_H_
|
