1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
|
//===-- hwasan_thread_list.h ------------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file is a part of HWAddressSanitizer.
//
//===----------------------------------------------------------------------===//
// HwasanThreadList is a registry for live threads, as well as an allocator for
// HwasanThread objects and their stack history ring buffers. There are
// constraints on memory layout of the shadow region and CompactRingBuffer that
// are part of the ABI contract between compiler-rt and llvm.
//
// * Start of the shadow memory region is aligned to 2**kShadowBaseAlignment.
// * All stack ring buffers are located within (2**kShadowBaseAlignment)
// sized region below and adjacent to the shadow region.
// * Each ring buffer has a size of (2**N)*4096 where N is in [0, 8), and is
// aligned to twice its size. The value of N can be different for each buffer.
//
// These constrains guarantee that, given an address A of any element of the
// ring buffer,
// A_next = (A + sizeof(uptr)) & ~((1 << (N + 13)) - 1)
// is the address of the next element of that ring buffer (with wrap-around).
// And, with K = kShadowBaseAlignment,
// S = (A | ((1 << K) - 1)) + 1
// (align up to kShadowBaseAlignment) is the start of the shadow region.
//
// These calculations are used in compiler instrumentation to update the ring
// buffer and obtain the base address of shadow using only two inputs: address
// of the current element of the ring buffer, and N (i.e. size of the ring
// buffer). Since the value of N is very limited, we pack both inputs into a
// single thread-local word as
// (1 << (N + 56)) | A
// See the implementation of class CompactRingBuffer, which is what is stored in
// said thread-local word.
//
// Note the unusual way of aligning up the address of the shadow:
// (A | ((1 << K) - 1)) + 1
// It is only correct if A is not already equal to the shadow base address, but
// it saves 2 instructions on AArch64.
#include "hwasan.h"
#include "hwasan_allocator.h"
#include "hwasan_flags.h"
#include "hwasan_thread.h"
#include "sanitizer_common/sanitizer_placement_new.h"
namespace __hwasan {
static uptr RingBufferSize() {
uptr desired_bytes = flags()->stack_history_size * sizeof(uptr);
// FIXME: increase the limit to 8 once this bug is fixed:
// https://bugs.llvm.org/show_bug.cgi?id=39030
for (int shift = 1; shift < 7; ++shift) {
uptr size = 4096 * (1ULL << shift);
if (size >= desired_bytes)
return size;
}
Printf("stack history size too large: %d\n", flags()->stack_history_size);
CHECK(0);
return 0;
}
struct ThreadStats {
uptr n_live_threads;
uptr total_stack_size;
};
class HwasanThreadList {
public:
HwasanThreadList(uptr storage, uptr size)
: free_space_(storage), free_space_end_(storage + size) {
// [storage, storage + size) is used as a vector of
// thread_alloc_size_-sized, ring_buffer_size_*2-aligned elements.
// Each element contains
// * a ring buffer at offset 0,
// * a Thread object at offset ring_buffer_size_.
ring_buffer_size_ = RingBufferSize();
thread_alloc_size_ =
RoundUpTo(ring_buffer_size_ + sizeof(Thread), ring_buffer_size_ * 2);
}
Thread *CreateCurrentThread(const Thread::InitState *state = nullptr) {
Thread *t = nullptr;
{
SpinMutexLock l(&free_list_mutex_);
if (!free_list_.empty()) {
t = free_list_.back();
free_list_.pop_back();
}
}
if (t) {
uptr start = (uptr)t - ring_buffer_size_;
internal_memset((void *)start, 0, ring_buffer_size_ + sizeof(Thread));
} else {
t = AllocThread();
}
{
SpinMutexLock l(&live_list_mutex_);
live_list_.push_back(t);
}
t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_, state);
AddThreadStats(t);
return t;
}
void DontNeedThread(Thread *t) {
uptr start = (uptr)t - ring_buffer_size_;
ReleaseMemoryPagesToOS(start, start + thread_alloc_size_);
}
void RemoveThreadFromLiveList(Thread *t) {
SpinMutexLock l(&live_list_mutex_);
for (Thread *&t2 : live_list_)
if (t2 == t) {
// To remove t2, copy the last element of the list in t2's position, and
// pop_back(). This works even if t2 is itself the last element.
t2 = live_list_.back();
live_list_.pop_back();
return;
}
CHECK(0 && "thread not found in live list");
}
void ReleaseThread(Thread *t) {
RemoveThreadStats(t);
t->Destroy();
DontNeedThread(t);
RemoveThreadFromLiveList(t);
SpinMutexLock l(&free_list_mutex_);
free_list_.push_back(t);
}
Thread *GetThreadByBufferAddress(uptr p) {
return (Thread *)(RoundDownTo(p, ring_buffer_size_ * 2) +
ring_buffer_size_);
}
uptr MemoryUsedPerThread() {
uptr res = sizeof(Thread) + ring_buffer_size_;
if (auto sz = flags()->heap_history_size)
res += HeapAllocationsRingBuffer::SizeInBytes(sz);
return res;
}
template <class CB>
void VisitAllLiveThreads(CB cb) {
SpinMutexLock l(&live_list_mutex_);
for (Thread *t : live_list_) cb(t);
}
void AddThreadStats(Thread *t) {
SpinMutexLock l(&stats_mutex_);
stats_.n_live_threads++;
stats_.total_stack_size += t->stack_size();
}
void RemoveThreadStats(Thread *t) {
SpinMutexLock l(&stats_mutex_);
stats_.n_live_threads--;
stats_.total_stack_size -= t->stack_size();
}
ThreadStats GetThreadStats() {
SpinMutexLock l(&stats_mutex_);
return stats_;
}
uptr GetRingBufferSize() const { return ring_buffer_size_; }
private:
Thread *AllocThread() {
SpinMutexLock l(&free_space_mutex_);
uptr align = ring_buffer_size_ * 2;
CHECK(IsAligned(free_space_, align));
Thread *t = (Thread *)(free_space_ + ring_buffer_size_);
free_space_ += thread_alloc_size_;
CHECK(free_space_ <= free_space_end_ && "out of thread memory");
return t;
}
SpinMutex free_space_mutex_;
uptr free_space_;
uptr free_space_end_;
uptr ring_buffer_size_;
uptr thread_alloc_size_;
SpinMutex free_list_mutex_;
InternalMmapVector<Thread *> free_list_;
SpinMutex live_list_mutex_;
InternalMmapVector<Thread *> live_list_;
ThreadStats stats_;
SpinMutex stats_mutex_;
};
void InitThreadList(uptr storage, uptr size);
HwasanThreadList &hwasanThreadList();
} // namespace __hwasan
|
