aboutsummaryrefslogtreecommitdiffstats
path: root/library/cpp/coroutine/engine
diff options
context:
space:
mode:
authoralexeylaptev <alexeylaptev@yandex-team.ru>2022-02-10 16:50:06 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:50:06 +0300
commit13dbd0acb78595551b843005d6bd021bdc1a859b (patch)
tree5d5cb817648f650d76cf1076100726fd9b8448e8 /library/cpp/coroutine/engine
parent3106abc0443cda789ce4968aeee63d3a8fcc7d85 (diff)
downloadydb-13dbd0acb78595551b843005d6bd021bdc1a859b.tar.gz
Restoring authorship annotation for <alexeylaptev@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'library/cpp/coroutine/engine')
-rw-r--r--library/cpp/coroutine/engine/impl.cpp34
-rw-r--r--library/cpp/coroutine/engine/impl.h24
-rw-r--r--library/cpp/coroutine/engine/stack/benchmark/alloc_bm.cpp632
-rw-r--r--library/cpp/coroutine/engine/stack/benchmark/ya.make26
-rw-r--r--library/cpp/coroutine/engine/stack/stack.cpp134
-rw-r--r--library/cpp/coroutine/engine/stack/stack.h154
-rw-r--r--library/cpp/coroutine/engine/stack/stack_allocator.cpp52
-rw-r--r--library/cpp/coroutine/engine/stack/stack_allocator.h104
-rw-r--r--library/cpp/coroutine/engine/stack/stack_allocator.inl276
-rw-r--r--library/cpp/coroutine/engine/stack/stack_common.h70
-rw-r--r--library/cpp/coroutine/engine/stack/stack_guards.cpp34
-rw-r--r--library/cpp/coroutine/engine/stack/stack_guards.h244
-rw-r--r--library/cpp/coroutine/engine/stack/stack_pool.h108
-rw-r--r--library/cpp/coroutine/engine/stack/stack_pool.inl264
-rw-r--r--library/cpp/coroutine/engine/stack/stack_storage.cpp92
-rw-r--r--library/cpp/coroutine/engine/stack/stack_storage.h120
-rw-r--r--library/cpp/coroutine/engine/stack/stack_utils.cpp168
-rw-r--r--library/cpp/coroutine/engine/stack/stack_utils.h54
-rw-r--r--library/cpp/coroutine/engine/stack/ut/stack_allocator_ut.cpp230
-rw-r--r--library/cpp/coroutine/engine/stack/ut/stack_guards_ut.cpp316
-rw-r--r--library/cpp/coroutine/engine/stack/ut/stack_pool_ut.cpp140
-rw-r--r--library/cpp/coroutine/engine/stack/ut/stack_ut.cpp120
-rw-r--r--library/cpp/coroutine/engine/stack/ut/stack_utils_ut.cpp146
-rw-r--r--library/cpp/coroutine/engine/stack/ut/ya.make34
-rw-r--r--library/cpp/coroutine/engine/trampoline.cpp8
-rw-r--r--library/cpp/coroutine/engine/trampoline.h16
-rw-r--r--library/cpp/coroutine/engine/ya.make12
27 files changed, 1806 insertions, 1806 deletions
diff --git a/library/cpp/coroutine/engine/impl.cpp b/library/cpp/coroutine/engine/impl.cpp
index 10fc6d9052..7ae6f74051 100644
--- a/library/cpp/coroutine/engine/impl.cpp
+++ b/library/cpp/coroutine/engine/impl.cpp
@@ -1,8 +1,8 @@
#include "impl.h"
-#include "stack/stack_allocator.h"
-#include "stack/stack_guards.h"
-
+#include "stack/stack_allocator.h"
+#include "stack/stack_guards.h"
+
#include <util/generic/scope.h>
#include <util/thread/singleton.h>
#include <util/stream/format.h>
@@ -18,15 +18,15 @@ void TCont::TJoinWait::Wake() noexcept {
Cont_.ReSchedule();
}
-TCont::TCont(NCoro::NStack::IAllocator& allocator,
- uint32_t stackSize,
- TContExecutor& executor,
+TCont::TCont(NCoro::NStack::IAllocator& allocator,
+ uint32_t stackSize,
+ TContExecutor& executor,
NCoro::TTrampoline::TFunc func,
- const char* name) noexcept
+ const char* name) noexcept
: Executor_(executor)
, Name_(name)
, Trampoline_(
- allocator,
+ allocator,
stackSize,
std::move(func),
this
@@ -117,11 +117,11 @@ void TCont::ReSchedule() noexcept {
TContExecutor::TContExecutor(
- uint32_t defaultStackSize,
+ uint32_t defaultStackSize,
THolder<IPollerFace> poller,
NCoro::IScheduleCallback* scheduleCallback,
NCoro::IEnterPollerCallback* enterPollerCallback,
- NCoro::NStack::EGuard defaultGuard,
+ NCoro::NStack::EGuard defaultGuard,
TMaybe<NCoro::NStack::TPoolAllocatorSettings> poolSettings,
NCoro::ITime* time
)
@@ -130,9 +130,9 @@ TContExecutor::TContExecutor(
, DefaultStackSize_(defaultStackSize)
, Poller_(std::move(poller))
, Time_(time)
-{
- StackAllocator_ = NCoro::NStack::GetAllocator(poolSettings, defaultGuard);
-}
+{
+ StackAllocator_ = NCoro::NStack::GetAllocator(poolSettings, defaultGuard);
+}
TContExecutor::~TContExecutor() {
Y_VERIFY(Allocated_ == 0, "leaked %u coroutines", (ui32)Allocated_);
@@ -253,10 +253,10 @@ TCont* TContExecutor::CreateOwned(
return cont;
}
-NCoro::NStack::TAllocatorStats TContExecutor::GetAllocatorStats() const noexcept {
- return StackAllocator_->GetStackStats();
-}
-
+NCoro::NStack::TAllocatorStats TContExecutor::GetAllocatorStats() const noexcept {
+ return StackAllocator_->GetStackStats();
+}
+
void TContExecutor::Release(TCont* cont) noexcept {
delete cont;
Allocated_ -= 1;
diff --git a/library/cpp/coroutine/engine/impl.h b/library/cpp/coroutine/engine/impl.h
index 4c33be254b..283a96ecf1 100644
--- a/library/cpp/coroutine/engine/impl.h
+++ b/library/cpp/coroutine/engine/impl.h
@@ -4,7 +4,7 @@
#include "cont_poller.h"
#include "iostatus.h"
#include "poller.h"
-#include "stack/stack_common.h"
+#include "stack/stack_common.h"
#include "trampoline.h"
#include "custom_time.h"
@@ -25,9 +25,9 @@ struct TContRep;
class TContExecutor;
class TContPollEvent;
-namespace NCoro::NStack {
- class IAllocator;
-}
+namespace NCoro::NStack {
+ class IAllocator;
+}
class TCont : private TIntrusiveListItem<TCont> {
struct TJoinWait: public TIntrusiveListItem<TJoinWait> {
@@ -46,8 +46,8 @@ class TCont : private TIntrusiveListItem<TCont> {
private:
TCont(
- NCoro::NStack::IAllocator& allocator,
- uint32_t stackSize,
+ NCoro::NStack::IAllocator& allocator,
+ uint32_t stackSize,
TContExecutor& executor,
NCoro::TTrampoline::TFunc func,
const char* name
@@ -151,11 +151,11 @@ class TContExecutor {
public:
TContExecutor(
- uint32_t defaultStackSize,
+ uint32_t defaultStackSize,
THolder<IPollerFace> poller = IPollerFace::Default(),
NCoro::IScheduleCallback* = nullptr,
NCoro::IEnterPollerCallback* = nullptr,
- NCoro::NStack::EGuard stackGuard = NCoro::NStack::EGuard::Canary,
+ NCoro::NStack::EGuard stackGuard = NCoro::NStack::EGuard::Canary,
TMaybe<NCoro::NStack::TPoolAllocatorSettings> poolSettings = Nothing(),
NCoro::ITime* time = nullptr
);
@@ -236,8 +236,8 @@ public:
return TotalConts() - TotalReadyConts();
}
- NCoro::NStack::TAllocatorStats GetAllocatorStats() const noexcept;
-
+ NCoro::NStack::TAllocatorStats GetAllocatorStats() const noexcept;
+
// TODO(velavokr): rename, it is just CancelAll actually
void Abort() noexcept;
@@ -290,8 +290,8 @@ private:
private:
NCoro::IScheduleCallback* const ScheduleCallback_ = nullptr;
NCoro::IEnterPollerCallback* const EnterPollerCallback_ = nullptr;
- const uint32_t DefaultStackSize_;
- THolder<NCoro::NStack::IAllocator> StackAllocator_;
+ const uint32_t DefaultStackSize_;
+ THolder<NCoro::NStack::IAllocator> StackAllocator_;
TExceptionSafeContext SchedContext_;
diff --git a/library/cpp/coroutine/engine/stack/benchmark/alloc_bm.cpp b/library/cpp/coroutine/engine/stack/benchmark/alloc_bm.cpp
index 927bccb310..38d713d274 100644
--- a/library/cpp/coroutine/engine/stack/benchmark/alloc_bm.cpp
+++ b/library/cpp/coroutine/engine/stack/benchmark/alloc_bm.cpp
@@ -1,316 +1,316 @@
-#include <benchmark/benchmark.h>
-
-#include <util/generic/vector.h>
-#include <util/system/yassert.h>
-
-#include <library/cpp/coroutine/engine/stack/stack_allocator.h>
-#include <library/cpp/coroutine/engine/stack/stack_guards.h>
-#include <library/cpp/coroutine/engine/stack/stack_pool.h>
-#include <library/cpp/coroutine/engine/stack/stack_utils.h>
-
-
-namespace NCoro::NStack::NBenchmark {
-
- const char* TestCoroName = "any_name";
- constexpr uint64_t BigCoroSize = PageSize * 25;
- constexpr uint64_t SmallCoroSize = PageSize * 4;
- constexpr uint64_t ManyStacks = 4096;
-
- void BasicOperations(TStackHolder& stack) {
- Y_VERIFY(!stack.Get().empty());
- stack.LowerCanaryOk();
- stack.UpperCanaryOk();
- }
-
- void WriteStack(TStackHolder& stack) {
- auto memory = stack.Get();
- Y_VERIFY(!memory.empty());
- stack.LowerCanaryOk();
- stack.UpperCanaryOk();
- for (uint64_t i = PageSize / 2; i < memory.size(); i += PageSize * 2) {
- memory[i] = 42;
- }
- }
-
- static void BM_GetAlignedMemory(benchmark::State& state) {
- char* raw = nullptr;
- char* aligned = nullptr;
- for (auto _ : state) {
- if (NCoro::NStack::GetAlignedMemory(state.range(0), raw, aligned)) {
- free(raw);
- }
- }
- }
- BENCHMARK(BM_GetAlignedMemory)->RangeMultiplier(16)->Range(1, 1024 * 1024);
-
- static void BM_GetAlignedMemoryReleaseRss(benchmark::State& state) {
- char* raw = nullptr;
- char* aligned = nullptr;
- for (auto _ : state) {
- if (NCoro::NStack::GetAlignedMemory(state.range(0), raw, aligned)) {
- const auto toFree = state.range(0) > 2 ? state.range(0) - 2 : 1;
- ReleaseRss(aligned, toFree);
- free(raw);
- }
- }
- }
- BENCHMARK(BM_GetAlignedMemoryReleaseRss)->RangeMultiplier(16)->Range(1, 1024 * 1024);
-
- static void BM_PoolAllocator(benchmark::State& state) {
- auto allocator = GetAllocator(TPoolAllocatorSettings{}, (EGuard)state.range(0));
- for (auto _ : state) {
- TStackHolder stack(*allocator, state.range(1), TestCoroName);
- BasicOperations(stack);
- }
- }
- BENCHMARK(BM_PoolAllocator)
- ->Args({(int64_t)EGuard::Canary, BigCoroSize}) // old version - ArgsProduct() is not supported
- ->Args({(int64_t)EGuard::Canary, SmallCoroSize})
- ->Args({(int64_t)EGuard::Page, BigCoroSize})
- ->Args({(int64_t)EGuard::Page, SmallCoroSize});
-
- static void BM_DefaultAllocator(benchmark::State& state) {
- auto allocator = GetAllocator(Nothing(), (EGuard)state.range(0));
- for (auto _ : state) {
- TStackHolder stack(*allocator, state.range(1), TestCoroName);
- BasicOperations(stack);
- }
- }
- BENCHMARK(BM_DefaultAllocator)
- ->Args({(int64_t)EGuard::Canary, BigCoroSize}) // old version - ArgsProduct() is not supported
- ->Args({(int64_t)EGuard::Canary, SmallCoroSize})
- ->Args({(int64_t)EGuard::Page, BigCoroSize})
- ->Args({(int64_t)EGuard::Page, SmallCoroSize});
-
- static void BM_PoolAllocatorManyStacksOneAtTime(benchmark::State& state) {
- TPoolAllocatorSettings settings;
- settings.StacksPerChunk = state.range(2);
- auto allocator = GetAllocator(settings, (EGuard)state.range(0));
- for (auto _ : state) {
- for (uint64_t i = 0; i < ManyStacks; ++i) {
- TStackHolder stack(*allocator, state.range(1), TestCoroName);
- BasicOperations(stack);
- }
- }
- }
- BENCHMARK(BM_PoolAllocatorManyStacksOneAtTime)
- ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1}) // old version - ArgsProduct() is not supported
- ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1})
- ->Args({(int64_t)EGuard::Page, BigCoroSize, 1})
- ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1})
- ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1024})
- ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1024})
- ->Args({(int64_t)EGuard::Page, BigCoroSize, 1024})
- ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1024});
-
- static void BM_DefaultAllocatorManyStacksOneAtTime(benchmark::State& state) {
- auto allocator = GetAllocator(Nothing(), (EGuard)state.range(0));
- for (auto _ : state) {
- for (uint64_t i = 0; i < ManyStacks; ++i) {
- TStackHolder stack(*allocator, state.range(1), TestCoroName);
- BasicOperations(stack);
- }
- }
- }
- BENCHMARK(BM_DefaultAllocatorManyStacksOneAtTime)
- ->Args({(int64_t)EGuard::Canary, BigCoroSize}) // old version - ArgsProduct() is not supported
- ->Args({(int64_t)EGuard::Canary, SmallCoroSize})
- ->Args({(int64_t)EGuard::Page, BigCoroSize})
- ->Args({(int64_t)EGuard::Page, SmallCoroSize});
-
- static void BM_PoolAllocatorManyStacks(benchmark::State& state) {
- TPoolAllocatorSettings settings;
- settings.StacksPerChunk = state.range(2);
- auto allocator = GetAllocator(settings, (EGuard)state.range(0));
- TVector<TStackHolder> stacks; // store stacks during benchmark
- stacks.reserve(ManyStacks);
- for (auto _ : state) {
- for (uint64_t i = 0; i < ManyStacks; ++i) {
- stacks.emplace_back(*allocator, state.range(1), TestCoroName);
- BasicOperations(stacks.back());
- }
- }
- }
- BENCHMARK(BM_PoolAllocatorManyStacks)
- ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1}) // old version - ArgsProduct() is not supported
- ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1})
- ->Args({(int64_t)EGuard::Page, BigCoroSize, 1})
- ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1})
- ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1024})
- ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1024})
- ->Args({(int64_t)EGuard::Page, BigCoroSize, 1024})
- ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1024});
-
- static void BM_DefaultAllocatorManyStacks(benchmark::State& state) {
- auto allocator = GetAllocator(Nothing(), (EGuard)state.range(0));
- TVector<TStackHolder> stacks; // store stacks during benchmark
- stacks.reserve(ManyStacks);
- for (auto _ : state) {
- for (uint64_t i = 0; i < ManyStacks; ++i) {
- stacks.push_back(TStackHolder(*allocator, state.range(1), TestCoroName));
- BasicOperations(stacks.back());
- }
- }
- }
- BENCHMARK(BM_DefaultAllocatorManyStacks)
- ->Args({(int64_t)EGuard::Canary, BigCoroSize}) // old version - ArgsProduct() is not supported
- ->Args({(int64_t)EGuard::Canary, SmallCoroSize})
- ->Args({(int64_t)EGuard::Page, BigCoroSize})
- ->Args({(int64_t)EGuard::Page, SmallCoroSize});
-
- // ------------------------------------------------------------------------
- static void BM_PoolAllocatorManyStacksReleased(benchmark::State& state) {
- TPoolAllocatorSettings settings;
- settings.StacksPerChunk = state.range(2);
- auto allocator = GetAllocator(settings, (EGuard)state.range(0));
- TVector<TStackHolder> stacks; // store stacks during benchmark
- stacks.reserve(ManyStacks);
- for (auto _ : state) {
- for (uint64_t i = 0; i < ManyStacks; ++i) {
- stacks.emplace_back(*allocator, state.range(1), TestCoroName);
- BasicOperations(stacks.back());
- }
- stacks.clear();
- }
- }
- BENCHMARK(BM_PoolAllocatorManyStacksReleased)
- ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1}) // old version - ArgsProduct() is not supported
- ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1})
- ->Args({(int64_t)EGuard::Page, BigCoroSize, 1})
- ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1})
- ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1024})
- ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1024})
- ->Args({(int64_t)EGuard::Page, BigCoroSize, 1024})
- ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1024});
-
- static void BM_DefaultAllocatorManyStacksReleased(benchmark::State& state) {
- auto allocator = GetAllocator(Nothing(), (EGuard)state.range(0));
- TVector<TStackHolder> stacks; // store stacks during benchmark
- stacks.reserve(ManyStacks);
- for (auto _ : state) {
- for (uint64_t i = 0; i < ManyStacks; ++i) {
- stacks.push_back(TStackHolder(*allocator, state.range(1), TestCoroName));
- BasicOperations(stacks.back());
- }
- stacks.clear();
- }
- }
- BENCHMARK(BM_DefaultAllocatorManyStacksReleased)
- ->Args({(int64_t)EGuard::Canary, BigCoroSize}) // old version - ArgsProduct() is not supported
- ->Args({(int64_t)EGuard::Canary, SmallCoroSize})
- ->Args({(int64_t)EGuard::Page, BigCoroSize})
- ->Args({(int64_t)EGuard::Page, SmallCoroSize});
-
- // ------------------------------------------------------------------------
- static void BM_PoolAllocatorManyStacksReleasedAndRealloc(benchmark::State& state) {
- TPoolAllocatorSettings settings;
- settings.StacksPerChunk = state.range(2);
- auto allocator = GetAllocator(settings, (EGuard)state.range(0));
- TVector<TStackHolder> stacks; // store stacks during benchmark
- stacks.reserve(ManyStacks);
- for (auto _ : state) {
- for (uint64_t i = 0; i < ManyStacks; ++i) {
- stacks.emplace_back(*allocator, state.range(1), TestCoroName);
- BasicOperations(stacks.back());
- }
- stacks.clear();
- for (uint64_t i = 0; i < ManyStacks; ++i) {
- stacks.emplace_back(*allocator, state.range(1), TestCoroName);
- BasicOperations(stacks.back());
- }
- }
- }
- BENCHMARK(BM_PoolAllocatorManyStacksReleasedAndRealloc)
- ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1}) // old version - ArgsProduct() is not supported
- ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1})
- ->Args({(int64_t)EGuard::Page, BigCoroSize, 1})
- ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1})
- ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1024})
- ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1024})
- ->Args({(int64_t)EGuard::Page, BigCoroSize, 1024})
- ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1024})
- ->Args({(int64_t)EGuard::Canary, BigCoroSize, 8192})
- ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 8192})
- ->Args({(int64_t)EGuard::Page, BigCoroSize, 8192})
- ->Args({(int64_t)EGuard::Page, SmallCoroSize, 8192});
-
- static void BM_DefaultAllocatorManyStacksReleasedAndRealloc(benchmark::State& state) {
- auto allocator = GetAllocator(Nothing(), (EGuard)state.range(0));
- TVector<TStackHolder> stacks; // store stacks during benchmark
- stacks.reserve(ManyStacks);
- for (auto _ : state) {
- for (uint64_t i = 0; i < ManyStacks; ++i) {
- stacks.push_back(TStackHolder(*allocator, state.range(1), TestCoroName));
- BasicOperations(stacks.back());
- }
- stacks.clear();
- for (uint64_t i = 0; i < ManyStacks; ++i) {
- stacks.push_back(TStackHolder(*allocator, state.range(1), TestCoroName));
- BasicOperations(stacks.back());
- }
- }
- }
- BENCHMARK(BM_DefaultAllocatorManyStacksReleasedAndRealloc)
- ->Args({(int64_t)EGuard::Canary, BigCoroSize}) // old version - ArgsProduct() is not supported
- ->Args({(int64_t)EGuard::Canary, SmallCoroSize})
- ->Args({(int64_t)EGuard::Page, BigCoroSize})
- ->Args({(int64_t)EGuard::Page, SmallCoroSize});
-
- // ------------------------------------------------------------------------
- static void BM_PoolAllocatorManyStacksMemoryWriteReleasedAndRealloc(benchmark::State& state) {
- TPoolAllocatorSettings settings;
- settings.StacksPerChunk = state.range(2);
- auto allocator = GetAllocator(settings, (EGuard)state.range(0));
- TVector<TStackHolder> stacks; // store stacks during benchmark
- stacks.reserve(ManyStacks);
- for (auto _ : state) {
- for (uint64_t i = 0; i < ManyStacks; ++i) {
- stacks.emplace_back(*allocator, state.range(1), TestCoroName);
- WriteStack(stacks.back());
- }
- stacks.clear();
- for (uint64_t i = 0; i < ManyStacks; ++i) {
- stacks.emplace_back(*allocator, state.range(1), TestCoroName);
- WriteStack(stacks.back());
- }
- }
- }
- BENCHMARK(BM_PoolAllocatorManyStacksMemoryWriteReleasedAndRealloc)
- ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1}) // old version - ArgsProduct() is not supported
- ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1})
- ->Args({(int64_t)EGuard::Page, BigCoroSize, 1})
- ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1})
- ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1024})
- ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1024})
- ->Args({(int64_t)EGuard::Page, BigCoroSize, 1024})
- ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1024})
- ->Args({(int64_t)EGuard::Canary, BigCoroSize, 8192})
- ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 8192})
- ->Args({(int64_t)EGuard::Page, BigCoroSize, 8192})
- ->Args({(int64_t)EGuard::Page, SmallCoroSize, 8192});
-
- static void BM_DefaultAllocatorManyStacksMemoryWriteReleasedAndRealloc(benchmark::State& state) {
- auto allocator = GetAllocator(Nothing(), (EGuard)state.range(0));
- TVector<TStackHolder> stacks; // store stacks during benchmark
- stacks.reserve(ManyStacks);
- for (auto _ : state) {
- for (uint64_t i = 0; i < ManyStacks; ++i) {
- stacks.push_back(TStackHolder(*allocator, state.range(1), TestCoroName));
- WriteStack(stacks.back());
- }
- stacks.clear();
- for (uint64_t i = 0; i < ManyStacks; ++i) {
- stacks.push_back(TStackHolder(*allocator, state.range(1), TestCoroName));
- WriteStack(stacks.back());
- }
- }
- }
- BENCHMARK(BM_DefaultAllocatorManyStacksMemoryWriteReleasedAndRealloc)
- ->Args({(int64_t)EGuard::Canary, BigCoroSize}) // old version - ArgsProduct() is not supported
- ->Args({(int64_t)EGuard::Canary, SmallCoroSize})
- ->Args({(int64_t)EGuard::Page, BigCoroSize})
- ->Args({(int64_t)EGuard::Page, SmallCoroSize});
-
-}
-
-BENCHMARK_MAIN();
+#include <benchmark/benchmark.h>
+
+#include <util/generic/vector.h>
+#include <util/system/yassert.h>
+
+#include <library/cpp/coroutine/engine/stack/stack_allocator.h>
+#include <library/cpp/coroutine/engine/stack/stack_guards.h>
+#include <library/cpp/coroutine/engine/stack/stack_pool.h>
+#include <library/cpp/coroutine/engine/stack/stack_utils.h>
+
+
+namespace NCoro::NStack::NBenchmark {
+
+ const char* TestCoroName = "any_name";
+ constexpr uint64_t BigCoroSize = PageSize * 25;
+ constexpr uint64_t SmallCoroSize = PageSize * 4;
+ constexpr uint64_t ManyStacks = 4096;
+
+ void BasicOperations(TStackHolder& stack) {
+ Y_VERIFY(!stack.Get().empty());
+ stack.LowerCanaryOk();
+ stack.UpperCanaryOk();
+ }
+
+ void WriteStack(TStackHolder& stack) {
+ auto memory = stack.Get();
+ Y_VERIFY(!memory.empty());
+ stack.LowerCanaryOk();
+ stack.UpperCanaryOk();
+ for (uint64_t i = PageSize / 2; i < memory.size(); i += PageSize * 2) {
+ memory[i] = 42;
+ }
+ }
+
+ static void BM_GetAlignedMemory(benchmark::State& state) {
+ char* raw = nullptr;
+ char* aligned = nullptr;
+ for (auto _ : state) {
+ if (NCoro::NStack::GetAlignedMemory(state.range(0), raw, aligned)) {
+ free(raw);
+ }
+ }
+ }
+ BENCHMARK(BM_GetAlignedMemory)->RangeMultiplier(16)->Range(1, 1024 * 1024);
+
+ static void BM_GetAlignedMemoryReleaseRss(benchmark::State& state) {
+ char* raw = nullptr;
+ char* aligned = nullptr;
+ for (auto _ : state) {
+ if (NCoro::NStack::GetAlignedMemory(state.range(0), raw, aligned)) {
+ const auto toFree = state.range(0) > 2 ? state.range(0) - 2 : 1;
+ ReleaseRss(aligned, toFree);
+ free(raw);
+ }
+ }
+ }
+ BENCHMARK(BM_GetAlignedMemoryReleaseRss)->RangeMultiplier(16)->Range(1, 1024 * 1024);
+
+ static void BM_PoolAllocator(benchmark::State& state) {
+ auto allocator = GetAllocator(TPoolAllocatorSettings{}, (EGuard)state.range(0));
+ for (auto _ : state) {
+ TStackHolder stack(*allocator, state.range(1), TestCoroName);
+ BasicOperations(stack);
+ }
+ }
+ BENCHMARK(BM_PoolAllocator)
+ ->Args({(int64_t)EGuard::Canary, BigCoroSize}) // old version - ArgsProduct() is not supported
+ ->Args({(int64_t)EGuard::Canary, SmallCoroSize})
+ ->Args({(int64_t)EGuard::Page, BigCoroSize})
+ ->Args({(int64_t)EGuard::Page, SmallCoroSize});
+
+ static void BM_DefaultAllocator(benchmark::State& state) {
+ auto allocator = GetAllocator(Nothing(), (EGuard)state.range(0));
+ for (auto _ : state) {
+ TStackHolder stack(*allocator, state.range(1), TestCoroName);
+ BasicOperations(stack);
+ }
+ }
+ BENCHMARK(BM_DefaultAllocator)
+ ->Args({(int64_t)EGuard::Canary, BigCoroSize}) // old version - ArgsProduct() is not supported
+ ->Args({(int64_t)EGuard::Canary, SmallCoroSize})
+ ->Args({(int64_t)EGuard::Page, BigCoroSize})
+ ->Args({(int64_t)EGuard::Page, SmallCoroSize});
+
+ static void BM_PoolAllocatorManyStacksOneAtTime(benchmark::State& state) {
+ TPoolAllocatorSettings settings;
+ settings.StacksPerChunk = state.range(2);
+ auto allocator = GetAllocator(settings, (EGuard)state.range(0));
+ for (auto _ : state) {
+ for (uint64_t i = 0; i < ManyStacks; ++i) {
+ TStackHolder stack(*allocator, state.range(1), TestCoroName);
+ BasicOperations(stack);
+ }
+ }
+ }
+ BENCHMARK(BM_PoolAllocatorManyStacksOneAtTime)
+ ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1}) // old version - ArgsProduct() is not supported
+ ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1})
+ ->Args({(int64_t)EGuard::Page, BigCoroSize, 1})
+ ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1})
+ ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1024})
+ ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1024})
+ ->Args({(int64_t)EGuard::Page, BigCoroSize, 1024})
+ ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1024});
+
+ static void BM_DefaultAllocatorManyStacksOneAtTime(benchmark::State& state) {
+ auto allocator = GetAllocator(Nothing(), (EGuard)state.range(0));
+ for (auto _ : state) {
+ for (uint64_t i = 0; i < ManyStacks; ++i) {
+ TStackHolder stack(*allocator, state.range(1), TestCoroName);
+ BasicOperations(stack);
+ }
+ }
+ }
+ BENCHMARK(BM_DefaultAllocatorManyStacksOneAtTime)
+ ->Args({(int64_t)EGuard::Canary, BigCoroSize}) // old version - ArgsProduct() is not supported
+ ->Args({(int64_t)EGuard::Canary, SmallCoroSize})
+ ->Args({(int64_t)EGuard::Page, BigCoroSize})
+ ->Args({(int64_t)EGuard::Page, SmallCoroSize});
+
+ static void BM_PoolAllocatorManyStacks(benchmark::State& state) {
+ TPoolAllocatorSettings settings;
+ settings.StacksPerChunk = state.range(2);
+ auto allocator = GetAllocator(settings, (EGuard)state.range(0));
+ TVector<TStackHolder> stacks; // store stacks during benchmark
+ stacks.reserve(ManyStacks);
+ for (auto _ : state) {
+ for (uint64_t i = 0; i < ManyStacks; ++i) {
+ stacks.emplace_back(*allocator, state.range(1), TestCoroName);
+ BasicOperations(stacks.back());
+ }
+ }
+ }
+ BENCHMARK(BM_PoolAllocatorManyStacks)
+ ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1}) // old version - ArgsProduct() is not supported
+ ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1})
+ ->Args({(int64_t)EGuard::Page, BigCoroSize, 1})
+ ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1})
+ ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1024})
+ ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1024})
+ ->Args({(int64_t)EGuard::Page, BigCoroSize, 1024})
+ ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1024});
+
+ static void BM_DefaultAllocatorManyStacks(benchmark::State& state) {
+ auto allocator = GetAllocator(Nothing(), (EGuard)state.range(0));
+ TVector<TStackHolder> stacks; // store stacks during benchmark
+ stacks.reserve(ManyStacks);
+ for (auto _ : state) {
+ for (uint64_t i = 0; i < ManyStacks; ++i) {
+ stacks.push_back(TStackHolder(*allocator, state.range(1), TestCoroName));
+ BasicOperations(stacks.back());
+ }
+ }
+ }
+ BENCHMARK(BM_DefaultAllocatorManyStacks)
+ ->Args({(int64_t)EGuard::Canary, BigCoroSize}) // old version - ArgsProduct() is not supported
+ ->Args({(int64_t)EGuard::Canary, SmallCoroSize})
+ ->Args({(int64_t)EGuard::Page, BigCoroSize})
+ ->Args({(int64_t)EGuard::Page, SmallCoroSize});
+
+ // ------------------------------------------------------------------------
+ static void BM_PoolAllocatorManyStacksReleased(benchmark::State& state) {
+ TPoolAllocatorSettings settings;
+ settings.StacksPerChunk = state.range(2);
+ auto allocator = GetAllocator(settings, (EGuard)state.range(0));
+ TVector<TStackHolder> stacks; // store stacks during benchmark
+ stacks.reserve(ManyStacks);
+ for (auto _ : state) {
+ for (uint64_t i = 0; i < ManyStacks; ++i) {
+ stacks.emplace_back(*allocator, state.range(1), TestCoroName);
+ BasicOperations(stacks.back());
+ }
+ stacks.clear();
+ }
+ }
+ BENCHMARK(BM_PoolAllocatorManyStacksReleased)
+ ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1}) // old version - ArgsProduct() is not supported
+ ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1})
+ ->Args({(int64_t)EGuard::Page, BigCoroSize, 1})
+ ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1})
+ ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1024})
+ ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1024})
+ ->Args({(int64_t)EGuard::Page, BigCoroSize, 1024})
+ ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1024});
+
+ static void BM_DefaultAllocatorManyStacksReleased(benchmark::State& state) {
+ auto allocator = GetAllocator(Nothing(), (EGuard)state.range(0));
+ TVector<TStackHolder> stacks; // store stacks during benchmark
+ stacks.reserve(ManyStacks);
+ for (auto _ : state) {
+ for (uint64_t i = 0; i < ManyStacks; ++i) {
+ stacks.push_back(TStackHolder(*allocator, state.range(1), TestCoroName));
+ BasicOperations(stacks.back());
+ }
+ stacks.clear();
+ }
+ }
+ BENCHMARK(BM_DefaultAllocatorManyStacksReleased)
+ ->Args({(int64_t)EGuard::Canary, BigCoroSize}) // old version - ArgsProduct() is not supported
+ ->Args({(int64_t)EGuard::Canary, SmallCoroSize})
+ ->Args({(int64_t)EGuard::Page, BigCoroSize})
+ ->Args({(int64_t)EGuard::Page, SmallCoroSize});
+
+ // ------------------------------------------------------------------------
+ static void BM_PoolAllocatorManyStacksReleasedAndRealloc(benchmark::State& state) {
+ TPoolAllocatorSettings settings;
+ settings.StacksPerChunk = state.range(2);
+ auto allocator = GetAllocator(settings, (EGuard)state.range(0));
+ TVector<TStackHolder> stacks; // store stacks during benchmark
+ stacks.reserve(ManyStacks);
+ for (auto _ : state) {
+ for (uint64_t i = 0; i < ManyStacks; ++i) {
+ stacks.emplace_back(*allocator, state.range(1), TestCoroName);
+ BasicOperations(stacks.back());
+ }
+ stacks.clear();
+ for (uint64_t i = 0; i < ManyStacks; ++i) {
+ stacks.emplace_back(*allocator, state.range(1), TestCoroName);
+ BasicOperations(stacks.back());
+ }
+ }
+ }
+ BENCHMARK(BM_PoolAllocatorManyStacksReleasedAndRealloc)
+ ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1}) // old version - ArgsProduct() is not supported
+ ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1})
+ ->Args({(int64_t)EGuard::Page, BigCoroSize, 1})
+ ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1})
+ ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1024})
+ ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1024})
+ ->Args({(int64_t)EGuard::Page, BigCoroSize, 1024})
+ ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1024})
+ ->Args({(int64_t)EGuard::Canary, BigCoroSize, 8192})
+ ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 8192})
+ ->Args({(int64_t)EGuard::Page, BigCoroSize, 8192})
+ ->Args({(int64_t)EGuard::Page, SmallCoroSize, 8192});
+
+ static void BM_DefaultAllocatorManyStacksReleasedAndRealloc(benchmark::State& state) {
+ auto allocator = GetAllocator(Nothing(), (EGuard)state.range(0));
+ TVector<TStackHolder> stacks; // store stacks during benchmark
+ stacks.reserve(ManyStacks);
+ for (auto _ : state) {
+ for (uint64_t i = 0; i < ManyStacks; ++i) {
+ stacks.push_back(TStackHolder(*allocator, state.range(1), TestCoroName));
+ BasicOperations(stacks.back());
+ }
+ stacks.clear();
+ for (uint64_t i = 0; i < ManyStacks; ++i) {
+ stacks.push_back(TStackHolder(*allocator, state.range(1), TestCoroName));
+ BasicOperations(stacks.back());
+ }
+ }
+ }
+ BENCHMARK(BM_DefaultAllocatorManyStacksReleasedAndRealloc)
+ ->Args({(int64_t)EGuard::Canary, BigCoroSize}) // old version - ArgsProduct() is not supported
+ ->Args({(int64_t)EGuard::Canary, SmallCoroSize})
+ ->Args({(int64_t)EGuard::Page, BigCoroSize})
+ ->Args({(int64_t)EGuard::Page, SmallCoroSize});
+
+ // ------------------------------------------------------------------------
+ static void BM_PoolAllocatorManyStacksMemoryWriteReleasedAndRealloc(benchmark::State& state) {
+ TPoolAllocatorSettings settings;
+ settings.StacksPerChunk = state.range(2);
+ auto allocator = GetAllocator(settings, (EGuard)state.range(0));
+ TVector<TStackHolder> stacks; // store stacks during benchmark
+ stacks.reserve(ManyStacks);
+ for (auto _ : state) {
+ for (uint64_t i = 0; i < ManyStacks; ++i) {
+ stacks.emplace_back(*allocator, state.range(1), TestCoroName);
+ WriteStack(stacks.back());
+ }
+ stacks.clear();
+ for (uint64_t i = 0; i < ManyStacks; ++i) {
+ stacks.emplace_back(*allocator, state.range(1), TestCoroName);
+ WriteStack(stacks.back());
+ }
+ }
+ }
+ BENCHMARK(BM_PoolAllocatorManyStacksMemoryWriteReleasedAndRealloc)
+ ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1}) // old version - ArgsProduct() is not supported
+ ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1})
+ ->Args({(int64_t)EGuard::Page, BigCoroSize, 1})
+ ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1})
+ ->Args({(int64_t)EGuard::Canary, BigCoroSize, 1024})
+ ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 1024})
+ ->Args({(int64_t)EGuard::Page, BigCoroSize, 1024})
+ ->Args({(int64_t)EGuard::Page, SmallCoroSize, 1024})
+ ->Args({(int64_t)EGuard::Canary, BigCoroSize, 8192})
+ ->Args({(int64_t)EGuard::Canary, SmallCoroSize, 8192})
+ ->Args({(int64_t)EGuard::Page, BigCoroSize, 8192})
+ ->Args({(int64_t)EGuard::Page, SmallCoroSize, 8192});
+
+ static void BM_DefaultAllocatorManyStacksMemoryWriteReleasedAndRealloc(benchmark::State& state) {
+ auto allocator = GetAllocator(Nothing(), (EGuard)state.range(0));
+ TVector<TStackHolder> stacks; // store stacks during benchmark
+ stacks.reserve(ManyStacks);
+ for (auto _ : state) {
+ for (uint64_t i = 0; i < ManyStacks; ++i) {
+ stacks.push_back(TStackHolder(*allocator, state.range(1), TestCoroName));
+ WriteStack(stacks.back());
+ }
+ stacks.clear();
+ for (uint64_t i = 0; i < ManyStacks; ++i) {
+ stacks.push_back(TStackHolder(*allocator, state.range(1), TestCoroName));
+ WriteStack(stacks.back());
+ }
+ }
+ }
+ BENCHMARK(BM_DefaultAllocatorManyStacksMemoryWriteReleasedAndRealloc)
+ ->Args({(int64_t)EGuard::Canary, BigCoroSize}) // old version - ArgsProduct() is not supported
+ ->Args({(int64_t)EGuard::Canary, SmallCoroSize})
+ ->Args({(int64_t)EGuard::Page, BigCoroSize})
+ ->Args({(int64_t)EGuard::Page, SmallCoroSize});
+
+}
+
+BENCHMARK_MAIN();
diff --git a/library/cpp/coroutine/engine/stack/benchmark/ya.make b/library/cpp/coroutine/engine/stack/benchmark/ya.make
index d14d75b341..b2942fe8ca 100644
--- a/library/cpp/coroutine/engine/stack/benchmark/ya.make
+++ b/library/cpp/coroutine/engine/stack/benchmark/ya.make
@@ -1,13 +1,13 @@
-G_BENCHMARK()
-
-OWNER(g:balancer)
-
-SRCS(
- alloc_bm.cpp
-)
-
-PEERDIR(
- library/cpp/coroutine/engine
-)
-
-END() \ No newline at end of file
+G_BENCHMARK()
+
+OWNER(g:balancer)
+
+SRCS(
+ alloc_bm.cpp
+)
+
+PEERDIR(
+ library/cpp/coroutine/engine
+)
+
+END() \ No newline at end of file
diff --git a/library/cpp/coroutine/engine/stack/stack.cpp b/library/cpp/coroutine/engine/stack/stack.cpp
index 658f263216..e29450261d 100644
--- a/library/cpp/coroutine/engine/stack/stack.cpp
+++ b/library/cpp/coroutine/engine/stack/stack.cpp
@@ -1,67 +1,67 @@
-#include "stack.h"
-
-#include "stack_allocator.h"
-#include "stack_guards.h"
-
-
-namespace NCoro::NStack {
-
-namespace NDetails {
-
- TStack::TStack(void* rawMemory, void* alignedMemory, uint64_t alignedSize, const char* /*name*/)
- : RawMemory_((char*)rawMemory)
- , AlignedMemory_((char*)alignedMemory)
- , Size_(alignedSize)
- {
- Y_ASSERT(AlignedMemory_ && RawMemory_ && Size_);
- Y_ASSERT(!(Size_ & PageSizeMask));
- Y_ASSERT(!((uint64_t)AlignedMemory_ & PageSizeMask));
- }
-
- TStack::TStack(TStack&& rhs) noexcept
- : RawMemory_(rhs.RawMemory_)
- , AlignedMemory_(rhs.AlignedMemory_)
- , Size_(rhs.Size_)
- {
- rhs.Reset();
- }
-
- TStack& TStack::operator=(TStack&& rhs) noexcept {
- std::swap(*this, rhs);
- rhs.Reset();
- return *this;
- }
-
- void TStack::Reset() noexcept {
- Y_ASSERT(AlignedMemory_ && RawMemory_ && Size_);
-
- RawMemory_ = nullptr;
- AlignedMemory_ = nullptr;
- Size_ = 0;
- }
-
-} // namespace NDetails
-
-
- TStackHolder::TStackHolder(NStack::IAllocator& allocator, uint32_t size, const char* name) noexcept
- : Allocator_(allocator)
- , Stack_(Allocator_.AllocStack(size, name))
- {}
-
- TStackHolder::~TStackHolder() {
- Allocator_.FreeStack(Stack_);
- }
-
- TArrayRef<char> TStackHolder::Get() noexcept {
- return Allocator_.GetStackWorkspace(Stack_.GetAlignedMemory(), Stack_.GetSize());
- }
-
- bool TStackHolder::LowerCanaryOk() const noexcept {
- return Allocator_.CheckStackOverflow(Stack_.GetAlignedMemory());
- }
-
- bool TStackHolder::UpperCanaryOk() const noexcept {
- return Allocator_.CheckStackOverride(Stack_.GetAlignedMemory(), Stack_.GetSize());
- }
-
-}
+#include "stack.h"
+
+#include "stack_allocator.h"
+#include "stack_guards.h"
+
+
+namespace NCoro::NStack {
+
+namespace NDetails {
+
+ TStack::TStack(void* rawMemory, void* alignedMemory, uint64_t alignedSize, const char* /*name*/)
+ : RawMemory_((char*)rawMemory)
+ , AlignedMemory_((char*)alignedMemory)
+ , Size_(alignedSize)
+ {
+ Y_ASSERT(AlignedMemory_ && RawMemory_ && Size_);
+ Y_ASSERT(!(Size_ & PageSizeMask));
+ Y_ASSERT(!((uint64_t)AlignedMemory_ & PageSizeMask));
+ }
+
+ TStack::TStack(TStack&& rhs) noexcept
+ : RawMemory_(rhs.RawMemory_)
+ , AlignedMemory_(rhs.AlignedMemory_)
+ , Size_(rhs.Size_)
+ {
+ rhs.Reset();
+ }
+
+ TStack& TStack::operator=(TStack&& rhs) noexcept {
+ std::swap(*this, rhs);
+ rhs.Reset();
+ return *this;
+ }
+
+ void TStack::Reset() noexcept {
+ Y_ASSERT(AlignedMemory_ && RawMemory_ && Size_);
+
+ RawMemory_ = nullptr;
+ AlignedMemory_ = nullptr;
+ Size_ = 0;
+ }
+
+} // namespace NDetails
+
+
+ TStackHolder::TStackHolder(NStack::IAllocator& allocator, uint32_t size, const char* name) noexcept
+ : Allocator_(allocator)
+ , Stack_(Allocator_.AllocStack(size, name))
+ {}
+
+ TStackHolder::~TStackHolder() {
+ Allocator_.FreeStack(Stack_);
+ }
+
+ TArrayRef<char> TStackHolder::Get() noexcept {
+ return Allocator_.GetStackWorkspace(Stack_.GetAlignedMemory(), Stack_.GetSize());
+ }
+
+ bool TStackHolder::LowerCanaryOk() const noexcept {
+ return Allocator_.CheckStackOverflow(Stack_.GetAlignedMemory());
+ }
+
+ bool TStackHolder::UpperCanaryOk() const noexcept {
+ return Allocator_.CheckStackOverride(Stack_.GetAlignedMemory(), Stack_.GetSize());
+ }
+
+}
diff --git a/library/cpp/coroutine/engine/stack/stack.h b/library/cpp/coroutine/engine/stack/stack.h
index e6b9a3a05d..7d98ba4c68 100644
--- a/library/cpp/coroutine/engine/stack/stack.h
+++ b/library/cpp/coroutine/engine/stack/stack.h
@@ -1,77 +1,77 @@
-#pragma once
-
-#include <util/generic/array_ref.h>
-#include <util/generic/fwd.h>
-#include <util/generic/noncopyable.h>
-
-#include <cstdint>
-
-
-namespace NCoro::NStack {
-
- class IAllocator;
-
-namespace NDetails {
-
- //! Do not use directly, use TStackHolder instead
- class TStack final : private TMoveOnly {
- public:
- /*! rawMemory: can be used by unaligned allocator to free stack memory after use
- * alignedMemory: pointer to aligned memory on which stack workspace and guard are actually placed
- * alignedSize: size of workspace memory + memory for guard
- * guard: guard to protect this stack
- * name: name of coroutine for which this stack is allocated
- */
- TStack(void* rawMemory, void* alignedMemory, uint64_t alignedSize, const char* name);
- TStack(TStack&& rhs) noexcept;
- TStack& operator=(TStack&& rhs) noexcept;
-
- char* GetRawMemory() const noexcept {
- return RawMemory_;
- }
-
- char* GetAlignedMemory() const noexcept {
- return AlignedMemory_;
- }
-
- //! Stack size (includes memory for guard)
- uint64_t GetSize() const noexcept {
- return Size_;
- }
-
- //! Resets parameters, should be called after stack memory is freed
- void Reset() noexcept;
-
- private:
- char* RawMemory_ = nullptr; // not owned
- char* AlignedMemory_ = nullptr; // not owned
- uint64_t Size_ = 0;
- };
-
-} // namespace NDetails
-
- class TStackHolder final : private TMoveOnly {
- public:
- explicit TStackHolder(IAllocator& allocator, uint32_t size, const char* name) noexcept;
- TStackHolder(TStackHolder&&) = default;
- TStackHolder& operator=(TStackHolder&&) = default;
-
- ~TStackHolder();
-
- char* GetAlignedMemory() const noexcept {
- return Stack_.GetAlignedMemory();
- }
- uint64_t GetSize() const noexcept {
- return Stack_.GetSize();
- }
-
- TArrayRef<char> Get() noexcept;
- bool LowerCanaryOk() const noexcept;
- bool UpperCanaryOk() const noexcept;
-
- private:
- IAllocator& Allocator_;
- NDetails::TStack Stack_;
- };
-
-}
+#pragma once
+
+#include <util/generic/array_ref.h>
+#include <util/generic/fwd.h>
+#include <util/generic/noncopyable.h>
+
+#include <cstdint>
+
+
+namespace NCoro::NStack {
+
+ class IAllocator;
+
+namespace NDetails {
+
+ //! Do not use directly, use TStackHolder instead
+ class TStack final : private TMoveOnly {
+ public:
+ /*! rawMemory: can be used by unaligned allocator to free stack memory after use
+ * alignedMemory: pointer to aligned memory on which stack workspace and guard are actually placed
+ * alignedSize: size of workspace memory + memory for guard
+ * guard: guard to protect this stack
+ * name: name of coroutine for which this stack is allocated
+ */
+ TStack(void* rawMemory, void* alignedMemory, uint64_t alignedSize, const char* name);
+ TStack(TStack&& rhs) noexcept;
+ TStack& operator=(TStack&& rhs) noexcept;
+
+ char* GetRawMemory() const noexcept {
+ return RawMemory_;
+ }
+
+ char* GetAlignedMemory() const noexcept {
+ return AlignedMemory_;
+ }
+
+ //! Stack size (includes memory for guard)
+ uint64_t GetSize() const noexcept {
+ return Size_;
+ }
+
+ //! Resets parameters, should be called after stack memory is freed
+ void Reset() noexcept;
+
+ private:
+ char* RawMemory_ = nullptr; // not owned
+ char* AlignedMemory_ = nullptr; // not owned
+ uint64_t Size_ = 0;
+ };
+
+} // namespace NDetails
+
+ class TStackHolder final : private TMoveOnly {
+ public:
+ explicit TStackHolder(IAllocator& allocator, uint32_t size, const char* name) noexcept;
+ TStackHolder(TStackHolder&&) = default;
+ TStackHolder& operator=(TStackHolder&&) = default;
+
+ ~TStackHolder();
+
+ char* GetAlignedMemory() const noexcept {
+ return Stack_.GetAlignedMemory();
+ }
+ uint64_t GetSize() const noexcept {
+ return Stack_.GetSize();
+ }
+
+ TArrayRef<char> Get() noexcept;
+ bool LowerCanaryOk() const noexcept;
+ bool UpperCanaryOk() const noexcept;
+
+ private:
+ IAllocator& Allocator_;
+ NDetails::TStack Stack_;
+ };
+
+}
diff --git a/library/cpp/coroutine/engine/stack/stack_allocator.cpp b/library/cpp/coroutine/engine/stack/stack_allocator.cpp
index 494f5d315a..bf12134e6b 100644
--- a/library/cpp/coroutine/engine/stack/stack_allocator.cpp
+++ b/library/cpp/coroutine/engine/stack/stack_allocator.cpp
@@ -1,26 +1,26 @@
-#include "stack_allocator.h"
-
-
-namespace NCoro::NStack {
-
- THolder<IAllocator> GetAllocator(TMaybe<TPoolAllocatorSettings> poolSettings, EGuard guardType) {
- THolder<IAllocator> allocator;
- if (poolSettings) {
- if (guardType == EGuard::Canary) {
- allocator = MakeHolder<TPoolAllocator<TCanaryGuard>>(*poolSettings);
- } else {
- Y_ASSERT(guardType == EGuard::Page);
- allocator = MakeHolder<TPoolAllocator<TPageGuard>>(*poolSettings);
- }
- } else {
- if (guardType == EGuard::Canary) {
- allocator = MakeHolder<TSimpleAllocator<TCanaryGuard>>();
- } else {
- Y_ASSERT(guardType == EGuard::Page);
- allocator = MakeHolder<TSimpleAllocator<TPageGuard>>();
- }
- }
- return allocator;
- }
-
-} \ No newline at end of file
+#include "stack_allocator.h"
+
+
+namespace NCoro::NStack {
+
+ THolder<IAllocator> GetAllocator(TMaybe<TPoolAllocatorSettings> poolSettings, EGuard guardType) {
+ THolder<IAllocator> allocator;
+ if (poolSettings) {
+ if (guardType == EGuard::Canary) {
+ allocator = MakeHolder<TPoolAllocator<TCanaryGuard>>(*poolSettings);
+ } else {
+ Y_ASSERT(guardType == EGuard::Page);
+ allocator = MakeHolder<TPoolAllocator<TPageGuard>>(*poolSettings);
+ }
+ } else {
+ if (guardType == EGuard::Canary) {
+ allocator = MakeHolder<TSimpleAllocator<TCanaryGuard>>();
+ } else {
+ Y_ASSERT(guardType == EGuard::Page);
+ allocator = MakeHolder<TSimpleAllocator<TPageGuard>>();
+ }
+ }
+ return allocator;
+ }
+
+} \ No newline at end of file
diff --git a/library/cpp/coroutine/engine/stack/stack_allocator.h b/library/cpp/coroutine/engine/stack/stack_allocator.h
index 543c715634..da3c3a93a1 100644
--- a/library/cpp/coroutine/engine/stack/stack_allocator.h
+++ b/library/cpp/coroutine/engine/stack/stack_allocator.h
@@ -1,52 +1,52 @@
-#pragma once
-
-#include "stack.h"
-#include "stack_common.h"
-
-#include <util/generic/maybe.h>
-#include <util/generic/noncopyable.h>
-#include <util/generic/ptr.h>
-
-#include <cstdint>
-
-
-namespace NCoro::NStack {
-
- class IAllocator : private TNonCopyable {
- public:
- virtual ~IAllocator() = default;
-
- //! Size should be page-aligned. Stack would be protected by guard, thus, actual
- //! workspace for stack = size - size of guard.
- NDetails::TStack AllocStack(uint64_t size, const char* name) {
- uint64_t alignedSize = (size + PageSize - 1) & ~PageSizeMask;
- Y_ASSERT(alignedSize < 10 * 1024 * PageSize); // more than 10K pages for stack - do you really need it?
-#if defined(_san_enabled_) || !defined(NDEBUG)
- alignedSize *= DebugOrSanStackMultiplier;
-#endif
- return DoAllocStack(alignedSize, name);
- }
-
- void FreeStack(NDetails::TStack& stack) noexcept {
- if (stack.GetAlignedMemory()) {
- DoFreeStack(stack);
- }
- };
-
- virtual TAllocatorStats GetStackStats() const noexcept = 0;
-
- // Stack helpers
- virtual TArrayRef<char> GetStackWorkspace(void* stack, uint64_t size) noexcept = 0;
- virtual bool CheckStackOverflow(void* stack) const noexcept = 0;
- virtual bool CheckStackOverride(void* stack, uint64_t size) const noexcept = 0;
-
- private:
- virtual NDetails::TStack DoAllocStack(uint64_t size, const char* name) = 0;
- virtual void DoFreeStack(NDetails::TStack& stack) noexcept = 0;
- };
-
- THolder<IAllocator> GetAllocator(TMaybe<TPoolAllocatorSettings> poolSettings, EGuard guardType);
-
-}
-
-#include "stack_allocator.inl"
+#pragma once
+
+#include "stack.h"
+#include "stack_common.h"
+
+#include <util/generic/maybe.h>
+#include <util/generic/noncopyable.h>
+#include <util/generic/ptr.h>
+
+#include <cstdint>
+
+
+namespace NCoro::NStack {
+
+ class IAllocator : private TNonCopyable {
+ public:
+ virtual ~IAllocator() = default;
+
+ //! Size should be page-aligned. Stack would be protected by guard, thus, actual
+ //! workspace for stack = size - size of guard.
+ NDetails::TStack AllocStack(uint64_t size, const char* name) {
+ uint64_t alignedSize = (size + PageSize - 1) & ~PageSizeMask;
+ Y_ASSERT(alignedSize < 10 * 1024 * PageSize); // more than 10K pages for stack - do you really need it?
+#if defined(_san_enabled_) || !defined(NDEBUG)
+ alignedSize *= DebugOrSanStackMultiplier;
+#endif
+ return DoAllocStack(alignedSize, name);
+ }
+
+ void FreeStack(NDetails::TStack& stack) noexcept {
+ if (stack.GetAlignedMemory()) {
+ DoFreeStack(stack);
+ }
+ };
+
+ virtual TAllocatorStats GetStackStats() const noexcept = 0;
+
+ // Stack helpers
+ virtual TArrayRef<char> GetStackWorkspace(void* stack, uint64_t size) noexcept = 0;
+ virtual bool CheckStackOverflow(void* stack) const noexcept = 0;
+ virtual bool CheckStackOverride(void* stack, uint64_t size) const noexcept = 0;
+
+ private:
+ virtual NDetails::TStack DoAllocStack(uint64_t size, const char* name) = 0;
+ virtual void DoFreeStack(NDetails::TStack& stack) noexcept = 0;
+ };
+
+ THolder<IAllocator> GetAllocator(TMaybe<TPoolAllocatorSettings> poolSettings, EGuard guardType);
+
+}
+
+#include "stack_allocator.inl"
diff --git a/library/cpp/coroutine/engine/stack/stack_allocator.inl b/library/cpp/coroutine/engine/stack/stack_allocator.inl
index b8bf83840f..0f25a4167b 100644
--- a/library/cpp/coroutine/engine/stack/stack_allocator.inl
+++ b/library/cpp/coroutine/engine/stack/stack_allocator.inl
@@ -1,138 +1,138 @@
-#include "stack_guards.h"
-#include "stack_pool.h"
-#include "stack_utils.h"
-
-#include <util/generic/hash.h>
-
-#ifdef _linux_
-#include <unistd.h>
-#endif
-
-
-namespace NCoro::NStack {
-
- template<typename TGuard>
- class TPoolAllocator final : public IAllocator {
- public:
- explicit TPoolAllocator(const TPoolAllocatorSettings& settings);
-
- TArrayRef<char> GetStackWorkspace(void* stack, uint64_t size) noexcept override {
- return Guard_.GetWorkspace(stack, size);
- }
- bool CheckStackOverflow(void* stack) const noexcept override {
- return Guard_.CheckOverflow(stack);
- }
- bool CheckStackOverride(void* stack, uint64_t size) const noexcept override {
- return Guard_.CheckOverride(stack, size);
- }
-
- TAllocatorStats GetStackStats() const noexcept override {
- TAllocatorStats stats;
- for (const auto& i : Pools_) {
- stats.ReleasedSize += i.second.GetReleasedSize();
- stats.NotReleasedSize += i.second.GetFullSize();
- stats.NumOfAllocated += i.second.GetNumOfAllocated();
- }
- return stats;
- }
-
- private: // methods
- NDetails::TStack DoAllocStack(uint64_t size, const char* name) override;
- void DoFreeStack(NDetails::TStack& stack) noexcept override;
-
- private: // data
- const TPoolAllocatorSettings PoolSettings_;
- const TGuard& Guard_;
- THashMap<uint64_t, TPool<TGuard>> Pools_; // key - stack size
- };
-
- template<typename TGuard>
- TPoolAllocator<TGuard>::TPoolAllocator(const TPoolAllocatorSettings& settings)
- : PoolSettings_(settings)
- , Guard_(GetGuard<TGuard>())
- {
-#ifdef _linux_
- Y_VERIFY(sysconf(_SC_PAGESIZE) == PageSize);
-#endif
- }
-
- template<typename TGuard>
- NDetails::TStack TPoolAllocator<TGuard>::DoAllocStack(uint64_t alignedSize, const char* name) {
- Y_ASSERT(alignedSize > Guard_.GetSize());
-
- auto pool = Pools_.find(alignedSize);
- if (pool == Pools_.end()) {
- Y_ASSERT(Pools_.size() < 1000); // too many different sizes for coroutine stacks
- auto [newPool, success] = Pools_.emplace(alignedSize, TPool<TGuard>{alignedSize, PoolSettings_, Guard_});
- Y_VERIFY(success, "Failed to add new coroutine pool");
- pool = newPool;
- }
- return pool->second.AllocStack(name);
- }
-
- template<typename TGuard>
- void TPoolAllocator<TGuard>::DoFreeStack(NDetails::TStack& stack) noexcept {
- auto pool = Pools_.find(stack.GetSize());
- Y_VERIFY(pool != Pools_.end(), "Attempt to free stack from another allocator");
- pool->second.FreeStack(stack);
- }
-
- // ------------------------------------------------------------------------
- //
- template<typename TGuard>
- class TSimpleAllocator : public IAllocator {
- public:
- explicit TSimpleAllocator();
-
- TArrayRef<char> GetStackWorkspace(void* stack, uint64_t size) noexcept override {
- return Guard_.GetWorkspace(stack, size);
- }
- bool CheckStackOverflow(void* stack) const noexcept override {
- return Guard_.CheckOverflow(stack);
- }
- bool CheckStackOverride(void* stack, uint64_t size) const noexcept override {
- return Guard_.CheckOverride(stack, size);
- }
-
- TAllocatorStats GetStackStats() const noexcept override { return {}; } // not used for simple allocator
-
- private: // methods
- NDetails::TStack DoAllocStack(uint64_t size, const char* name) override;
- void DoFreeStack(NDetails::TStack& stack) noexcept override;
-
- private: // data
- const TGuard& Guard_;
- };
-
-
- template<typename TGuard>
- TSimpleAllocator<TGuard>::TSimpleAllocator()
- : Guard_(GetGuard<TGuard>())
- {}
-
- template<typename TGuard>
- NDetails::TStack TSimpleAllocator<TGuard>::DoAllocStack(uint64_t alignedSize, const char* name) {
- Y_ASSERT(alignedSize > Guard_.GetSize());
-
- char* rawPtr = nullptr;
- char* alignedPtr = nullptr; // with extra space for previous guard in this type of allocator
-
- Y_VERIFY(GetAlignedMemory((alignedSize + Guard_.GetPageAlignedSize()) / PageSize, rawPtr, alignedPtr)); // + memory for previous guard
- char* alignedStackMemory = alignedPtr + Guard_.GetPageAlignedSize(); // after previous guard
-
- // Default allocator sets both guards, because it doesn't have memory chunk with previous stack and guard on it
- Guard_.Protect((void*)alignedPtr, Guard_.GetPageAlignedSize(), false); // first guard should be before stack memory
- Guard_.Protect(alignedStackMemory, alignedSize, true); // second guard is placed on stack memory
-
- return NDetails::TStack{rawPtr, alignedStackMemory, alignedSize, name};
- }
-
- template<typename TGuard>
- void TSimpleAllocator<TGuard>::DoFreeStack(NDetails::TStack& stack) noexcept {
- Guard_.RemoveProtection(stack.GetAlignedMemory() - Guard_.GetPageAlignedSize(), Guard_.GetPageAlignedSize());
- Guard_.RemoveProtection(stack.GetAlignedMemory(), stack.GetSize());
-
- free(stack.GetRawMemory());
- stack.Reset();
- }
-}
+#include "stack_guards.h"
+#include "stack_pool.h"
+#include "stack_utils.h"
+
+#include <util/generic/hash.h>
+
+#ifdef _linux_
+#include <unistd.h>
+#endif
+
+
+namespace NCoro::NStack {
+
+ template<typename TGuard>
+ class TPoolAllocator final : public IAllocator {
+ public:
+ explicit TPoolAllocator(const TPoolAllocatorSettings& settings);
+
+ TArrayRef<char> GetStackWorkspace(void* stack, uint64_t size) noexcept override {
+ return Guard_.GetWorkspace(stack, size);
+ }
+ bool CheckStackOverflow(void* stack) const noexcept override {
+ return Guard_.CheckOverflow(stack);
+ }
+ bool CheckStackOverride(void* stack, uint64_t size) const noexcept override {
+ return Guard_.CheckOverride(stack, size);
+ }
+
+ TAllocatorStats GetStackStats() const noexcept override {
+ TAllocatorStats stats;
+ for (const auto& i : Pools_) {
+ stats.ReleasedSize += i.second.GetReleasedSize();
+ stats.NotReleasedSize += i.second.GetFullSize();
+ stats.NumOfAllocated += i.second.GetNumOfAllocated();
+ }
+ return stats;
+ }
+
+ private: // methods
+ NDetails::TStack DoAllocStack(uint64_t size, const char* name) override;
+ void DoFreeStack(NDetails::TStack& stack) noexcept override;
+
+ private: // data
+ const TPoolAllocatorSettings PoolSettings_;
+ const TGuard& Guard_;
+ THashMap<uint64_t, TPool<TGuard>> Pools_; // key - stack size
+ };
+
+ template<typename TGuard>
+ TPoolAllocator<TGuard>::TPoolAllocator(const TPoolAllocatorSettings& settings)
+ : PoolSettings_(settings)
+ , Guard_(GetGuard<TGuard>())
+ {
+#ifdef _linux_
+ Y_VERIFY(sysconf(_SC_PAGESIZE) == PageSize);
+#endif
+ }
+
+ template<typename TGuard>
+ NDetails::TStack TPoolAllocator<TGuard>::DoAllocStack(uint64_t alignedSize, const char* name) {
+ Y_ASSERT(alignedSize > Guard_.GetSize());
+
+ auto pool = Pools_.find(alignedSize);
+ if (pool == Pools_.end()) {
+ Y_ASSERT(Pools_.size() < 1000); // too many different sizes for coroutine stacks
+ auto [newPool, success] = Pools_.emplace(alignedSize, TPool<TGuard>{alignedSize, PoolSettings_, Guard_});
+ Y_VERIFY(success, "Failed to add new coroutine pool");
+ pool = newPool;
+ }
+ return pool->second.AllocStack(name);
+ }
+
+ template<typename TGuard>
+ void TPoolAllocator<TGuard>::DoFreeStack(NDetails::TStack& stack) noexcept {
+ auto pool = Pools_.find(stack.GetSize());
+ Y_VERIFY(pool != Pools_.end(), "Attempt to free stack from another allocator");
+ pool->second.FreeStack(stack);
+ }
+
+ // ------------------------------------------------------------------------
+ //
+ template<typename TGuard>
+ class TSimpleAllocator : public IAllocator {
+ public:
+ explicit TSimpleAllocator();
+
+ TArrayRef<char> GetStackWorkspace(void* stack, uint64_t size) noexcept override {
+ return Guard_.GetWorkspace(stack, size);
+ }
+ bool CheckStackOverflow(void* stack) const noexcept override {
+ return Guard_.CheckOverflow(stack);
+ }
+ bool CheckStackOverride(void* stack, uint64_t size) const noexcept override {
+ return Guard_.CheckOverride(stack, size);
+ }
+
+ TAllocatorStats GetStackStats() const noexcept override { return {}; } // not used for simple allocator
+
+ private: // methods
+ NDetails::TStack DoAllocStack(uint64_t size, const char* name) override;
+ void DoFreeStack(NDetails::TStack& stack) noexcept override;
+
+ private: // data
+ const TGuard& Guard_;
+ };
+
+
+ template<typename TGuard>
+ TSimpleAllocator<TGuard>::TSimpleAllocator()
+ : Guard_(GetGuard<TGuard>())
+ {}
+
+ template<typename TGuard>
+ NDetails::TStack TSimpleAllocator<TGuard>::DoAllocStack(uint64_t alignedSize, const char* name) {
+ Y_ASSERT(alignedSize > Guard_.GetSize());
+
+ char* rawPtr = nullptr;
+ char* alignedPtr = nullptr; // with extra space for previous guard in this type of allocator
+
+ Y_VERIFY(GetAlignedMemory((alignedSize + Guard_.GetPageAlignedSize()) / PageSize, rawPtr, alignedPtr)); // + memory for previous guard
+ char* alignedStackMemory = alignedPtr + Guard_.GetPageAlignedSize(); // after previous guard
+
+ // Default allocator sets both guards, because it doesn't have memory chunk with previous stack and guard on it
+ Guard_.Protect((void*)alignedPtr, Guard_.GetPageAlignedSize(), false); // first guard should be before stack memory
+ Guard_.Protect(alignedStackMemory, alignedSize, true); // second guard is placed on stack memory
+
+ return NDetails::TStack{rawPtr, alignedStackMemory, alignedSize, name};
+ }
+
+ template<typename TGuard>
+ void TSimpleAllocator<TGuard>::DoFreeStack(NDetails::TStack& stack) noexcept {
+ Guard_.RemoveProtection(stack.GetAlignedMemory() - Guard_.GetPageAlignedSize(), Guard_.GetPageAlignedSize());
+ Guard_.RemoveProtection(stack.GetAlignedMemory(), stack.GetSize());
+
+ free(stack.GetRawMemory());
+ stack.Reset();
+ }
+}
diff --git a/library/cpp/coroutine/engine/stack/stack_common.h b/library/cpp/coroutine/engine/stack/stack_common.h
index 49e3466956..ed2d74d296 100644
--- a/library/cpp/coroutine/engine/stack/stack_common.h
+++ b/library/cpp/coroutine/engine/stack/stack_common.h
@@ -1,35 +1,35 @@
-#pragma once
-
-#include <cstdint>
-
-class TContExecutor;
-
-namespace NCoro::NStack {
-
- static constexpr uint64_t PageSize = 4096;
- static constexpr uint64_t PageSizeMask = PageSize - 1; // for checks
- static constexpr uint64_t DebugOrSanStackMultiplier = 4; // for debug or sanitizer builds
- static constexpr uint64_t SmallStackMaxSizeInPages = 6;
-
- enum class EGuard {
- Canary, //!< writes some data to check it for corruption
- Page, //!< prohibits access to page memory
- };
-
- struct TPoolAllocatorSettings {
- uint64_t RssPagesToKeep = 1;
- uint64_t SmallStackRssPagesToKeep = 1; // for stack less than SmallStackMaxSizeInPages
- uint64_t ReleaseRate = 2;
-#if !defined(_san_enabled_) && defined(NDEBUG)
- uint64_t StacksPerChunk = 256;
-#else
- uint64_t StacksPerChunk = 2;
-#endif
- };
-
- struct TAllocatorStats {
- uint64_t ReleasedSize = 0;
- uint64_t NotReleasedSize = 0;
- uint64_t NumOfAllocated = 0;
- };
-}
+#pragma once
+
+#include <cstdint>
+
+class TContExecutor;
+
+namespace NCoro::NStack {
+
+ static constexpr uint64_t PageSize = 4096;
+ static constexpr uint64_t PageSizeMask = PageSize - 1; // for checks
+ static constexpr uint64_t DebugOrSanStackMultiplier = 4; // for debug or sanitizer builds
+ static constexpr uint64_t SmallStackMaxSizeInPages = 6;
+
+ enum class EGuard {
+ Canary, //!< writes some data to check it for corruption
+ Page, //!< prohibits access to page memory
+ };
+
+ struct TPoolAllocatorSettings {
+ uint64_t RssPagesToKeep = 1;
+ uint64_t SmallStackRssPagesToKeep = 1; // for stack less than SmallStackMaxSizeInPages
+ uint64_t ReleaseRate = 2;
+#if !defined(_san_enabled_) && defined(NDEBUG)
+ uint64_t StacksPerChunk = 256;
+#else
+ uint64_t StacksPerChunk = 2;
+#endif
+ };
+
+ struct TAllocatorStats {
+ uint64_t ReleasedSize = 0;
+ uint64_t NotReleasedSize = 0;
+ uint64_t NumOfAllocated = 0;
+ };
+}
diff --git a/library/cpp/coroutine/engine/stack/stack_guards.cpp b/library/cpp/coroutine/engine/stack/stack_guards.cpp
index 07d10ee300..b8bcff039e 100644
--- a/library/cpp/coroutine/engine/stack/stack_guards.cpp
+++ b/library/cpp/coroutine/engine/stack/stack_guards.cpp
@@ -1,17 +1,17 @@
-#include "stack_guards.h"
-
-
-namespace NCoro::NStack {
-
- template<>
- const TCanaryGuard& GetGuard<TCanaryGuard>() noexcept {
- static const TCanaryGuard guard;
- return guard;
- }
-
- template<>
- const TPageGuard& GetGuard<TPageGuard>() noexcept {
- static const TPageGuard guard;
- return guard;
- }
-} \ No newline at end of file
+#include "stack_guards.h"
+
+
+namespace NCoro::NStack {
+
+ template<>
+ const TCanaryGuard& GetGuard<TCanaryGuard>() noexcept {
+ static const TCanaryGuard guard;
+ return guard;
+ }
+
+ template<>
+ const TPageGuard& GetGuard<TPageGuard>() noexcept {
+ static const TPageGuard guard;
+ return guard;
+ }
+} \ No newline at end of file
diff --git a/library/cpp/coroutine/engine/stack/stack_guards.h b/library/cpp/coroutine/engine/stack/stack_guards.h
index c354339ed2..3a7ef26481 100644
--- a/library/cpp/coroutine/engine/stack/stack_guards.h
+++ b/library/cpp/coroutine/engine/stack/stack_guards.h
@@ -1,123 +1,123 @@
-#pragma once
-
-#include "stack_common.h"
-
-#include <util/generic/array_ref.h>
-#include <util/generic/strbuf.h>
-#include <util/system/protect.h>
-
-
-namespace NCoro::NStack {
-
- /*! Guard detect stack overflow/override, by setting memory before and after stack with predefined values/properties.
- * Actually, it sets memory only after the end of stack workspace memory - previous guard section should be set
- * already (for previous stack in case of pool allocator) and can be checked on demand.
- * Stack pointer should be page-aligned.
- */
-
-
- //! Checks integrity by writing a predefined sequence and comparing it with original
- class TCanaryGuard final {
- public:
- //! Size of guard section in bytes
- static constexpr uint64_t GetSize() { return Canary.size(); }
- //! Size of page-aligned guard section in bytes
- static constexpr uint64_t GetPageAlignedSize() { return AlignedSize_; }
-
- //! Get stack memory between guard sections
- static TArrayRef<char> GetWorkspace(void* stack, uint64_t size) noexcept {
- Y_ASSERT( !((uint64_t)stack & PageSizeMask) );
- Y_ASSERT( !(size & PageSizeMask) );
- Y_ASSERT(size > Canary.size());
-
- return {(char*) stack, size - Canary.size()};
- }
-
- /*! Set guard section before the end of stack memory (at stack + size - guard size position)
- * checkPrevious: check guard before stack memory for integrity
- */
- static void Protect(void* stack, uint64_t size, bool checkPrevious) noexcept {
- Y_ASSERT( !((uint64_t)stack & PageSizeMask) ); // stack pointer should be page aligned
- Y_ASSERT( !(size & PageSizeMask) ); // stack size should be page aligned
- Y_ASSERT(size >= Canary.size()); // stack should have enough space to place guard
-
- if (checkPrevious) {
- Y_VERIFY(CheckOverflow(stack), "Previous stack was corrupted");
- }
- auto guardPos = (char*) stack + size - Canary.size();
- memcpy(guardPos, Canary.data(), Canary.size());
- }
-
- //! This guard doesn't change memory flags
- static constexpr void RemoveProtection(void*, uint64_t) {}
- //! Should remove protection before returning memory to system
- static constexpr bool ShouldRemoveProtectionBeforeFree() { return false; }
-
- static bool CheckOverflow(void* stack) noexcept {
- Y_ASSERT(stack);
-
- char* guardPos = (char*) ((uint64_t)stack - Canary.size());
- return TStringBuf(guardPos, Canary.size()) == Canary;
- }
-
- static bool CheckOverride(void* stack, uint64_t size) noexcept {
- Y_ASSERT(stack);
- Y_ASSERT(size > Canary.size());
-
- char* guardPos = (char*) ((uint64_t)stack + size - Canary.size());
- return TStringBuf(guardPos, Canary.size()) == Canary;
- }
-
- private:
+#pragma once
+
+#include "stack_common.h"
+
+#include <util/generic/array_ref.h>
+#include <util/generic/strbuf.h>
+#include <util/system/protect.h>
+
+
+namespace NCoro::NStack {
+
+ /*! Guard detect stack overflow/override, by setting memory before and after stack with predefined values/properties.
+ * Actually, it sets memory only after the end of stack workspace memory - previous guard section should be set
+ * already (for previous stack in case of pool allocator) and can be checked on demand.
+ * Stack pointer should be page-aligned.
+ */
+
+
+ //! Checks integrity by writing a predefined sequence and comparing it with original
+ class TCanaryGuard final {
+ public:
+ //! Size of guard section in bytes
+ static constexpr uint64_t GetSize() { return Canary.size(); }
+ //! Size of page-aligned guard section in bytes
+ static constexpr uint64_t GetPageAlignedSize() { return AlignedSize_; }
+
+ //! Get stack memory between guard sections
+ static TArrayRef<char> GetWorkspace(void* stack, uint64_t size) noexcept {
+ Y_ASSERT( !((uint64_t)stack & PageSizeMask) );
+ Y_ASSERT( !(size & PageSizeMask) );
+ Y_ASSERT(size > Canary.size());
+
+ return {(char*) stack, size - Canary.size()};
+ }
+
+ /*! Set guard section before the end of stack memory (at stack + size - guard size position)
+ * checkPrevious: check guard before stack memory for integrity
+ */
+ static void Protect(void* stack, uint64_t size, bool checkPrevious) noexcept {
+ Y_ASSERT( !((uint64_t)stack & PageSizeMask) ); // stack pointer should be page aligned
+ Y_ASSERT( !(size & PageSizeMask) ); // stack size should be page aligned
+ Y_ASSERT(size >= Canary.size()); // stack should have enough space to place guard
+
+ if (checkPrevious) {
+ Y_VERIFY(CheckOverflow(stack), "Previous stack was corrupted");
+ }
+ auto guardPos = (char*) stack + size - Canary.size();
+ memcpy(guardPos, Canary.data(), Canary.size());
+ }
+
+ //! This guard doesn't change memory flags
+ static constexpr void RemoveProtection(void*, uint64_t) {}
+ //! Should remove protection before returning memory to system
+ static constexpr bool ShouldRemoveProtectionBeforeFree() { return false; }
+
+ static bool CheckOverflow(void* stack) noexcept {
+ Y_ASSERT(stack);
+
+ char* guardPos = (char*) ((uint64_t)stack - Canary.size());
+ return TStringBuf(guardPos, Canary.size()) == Canary;
+ }
+
+ static bool CheckOverride(void* stack, uint64_t size) noexcept {
+ Y_ASSERT(stack);
+ Y_ASSERT(size > Canary.size());
+
+ char* guardPos = (char*) ((uint64_t)stack + size - Canary.size());
+ return TStringBuf(guardPos, Canary.size()) == Canary;
+ }
+
+ private:
static constexpr TStringBuf Canary = "[ThisIsACanaryCoroutineStackGuardIfYouReadThisTheStackIsStillOK]";
- static_assert(Canary.size() == 64);
- static constexpr uint64_t AlignedSize_ = (Canary.size() + PageSize - 1) & ~PageSizeMask;
- };
-
-
- // ------------------------------------------------------------------------
- //
- //! Ensures integrity by removing access rights for border pages
- class TPageGuard final {
- public:
- //! Size of guard section in bytes
- static constexpr uint64_t GetSize() { return PageSize; }
- //! Size of page-aligned guard section in bytes
- static constexpr uint64_t GetPageAlignedSize() { return PageSize; }
-
- static TArrayRef<char> GetWorkspace(void* stack, uint64_t size) noexcept {
- Y_ASSERT( !((uint64_t)stack & PageSizeMask) );
- Y_ASSERT( !(size & PageSizeMask) );
- Y_ASSERT(size > PageSize);
-
- return {(char*)stack, size - PageSize};
- }
-
- static void Protect(void* stack, uint64_t size, bool /*checkPrevious*/) noexcept {
- Y_ASSERT( !((uint64_t)stack & PageSizeMask) ); // stack pointer should be page aligned
- Y_ASSERT( !(size & PageSizeMask) ); // stack size should be page aligned
- Y_ASSERT(size >= PageSize); // stack should have enough space to place guard
-
- ProtectMemory((char*)stack + size - PageSize, PageSize, PM_NONE);
- }
-
- //! Remove protection, to allow stack memory be freed
- static void RemoveProtection(void* stack, uint64_t size) noexcept {
- Y_ASSERT( !((uint64_t)stack & PageSizeMask) );
- Y_ASSERT( !(size & PageSizeMask) );
- Y_ASSERT(size >= PageSize);
-
- ProtectMemory((char*)stack + size - PageSize, PageSize, PM_WRITE | PM_READ);
- }
- //! Should remove protection before returning memory to system
- static constexpr bool ShouldRemoveProtectionBeforeFree() { return true; }
-
- //! For page guard is not used - it crashes process at once in this case.
- static constexpr bool CheckOverflow(void*) { return true; }
- static constexpr bool CheckOverride(void*, uint64_t) { return true; }
- };
-
-
- template<typename TGuard>
- const TGuard& GetGuard() noexcept;
-}
+ static_assert(Canary.size() == 64);
+ static constexpr uint64_t AlignedSize_ = (Canary.size() + PageSize - 1) & ~PageSizeMask;
+ };
+
+
+ // ------------------------------------------------------------------------
+ //
+ //! Ensures integrity by removing access rights for border pages
+ class TPageGuard final {
+ public:
+ //! Size of guard section in bytes
+ static constexpr uint64_t GetSize() { return PageSize; }
+ //! Size of page-aligned guard section in bytes
+ static constexpr uint64_t GetPageAlignedSize() { return PageSize; }
+
+ static TArrayRef<char> GetWorkspace(void* stack, uint64_t size) noexcept {
+ Y_ASSERT( !((uint64_t)stack & PageSizeMask) );
+ Y_ASSERT( !(size & PageSizeMask) );
+ Y_ASSERT(size > PageSize);
+
+ return {(char*)stack, size - PageSize};
+ }
+
+ static void Protect(void* stack, uint64_t size, bool /*checkPrevious*/) noexcept {
+ Y_ASSERT( !((uint64_t)stack & PageSizeMask) ); // stack pointer should be page aligned
+ Y_ASSERT( !(size & PageSizeMask) ); // stack size should be page aligned
+ Y_ASSERT(size >= PageSize); // stack should have enough space to place guard
+
+ ProtectMemory((char*)stack + size - PageSize, PageSize, PM_NONE);
+ }
+
+ //! Remove protection, to allow stack memory be freed
+ static void RemoveProtection(void* stack, uint64_t size) noexcept {
+ Y_ASSERT( !((uint64_t)stack & PageSizeMask) );
+ Y_ASSERT( !(size & PageSizeMask) );
+ Y_ASSERT(size >= PageSize);
+
+ ProtectMemory((char*)stack + size - PageSize, PageSize, PM_WRITE | PM_READ);
+ }
+ //! Should remove protection before returning memory to system
+ static constexpr bool ShouldRemoveProtectionBeforeFree() { return true; }
+
+ //! For page guard is not used - it crashes process at once in this case.
+ static constexpr bool CheckOverflow(void*) { return true; }
+ static constexpr bool CheckOverride(void*, uint64_t) { return true; }
+ };
+
+
+ template<typename TGuard>
+ const TGuard& GetGuard() noexcept;
+}
diff --git a/library/cpp/coroutine/engine/stack/stack_pool.h b/library/cpp/coroutine/engine/stack/stack_pool.h
index 0625362d4d..27a8e9394b 100644
--- a/library/cpp/coroutine/engine/stack/stack_pool.h
+++ b/library/cpp/coroutine/engine/stack/stack_pool.h
@@ -1,54 +1,54 @@
-#pragma once
-
-#include "stack.h"
-#include "stack_common.h"
-
-#include <util/generic/noncopyable.h>
-#include <util/generic/ptr.h>
-#include <util/generic/vector.h>
-
-
-namespace NCoro::NStack {
-
- class IGuard;
- class TStorage;
- struct TPoolAllocatorSettings;
-
- template<typename TGuard>
- class TPool final : private TMoveOnly {
- struct TMemory {
- char* Raw = nullptr;
- char* Aligned = nullptr; // points to aligned memory, which includes space for first page guard
- };
- public:
- TPool(uint64_t stackSize, const TPoolAllocatorSettings& settings, const TGuard& guard);
- TPool(TPool&& other) noexcept;
- ~TPool();
-
- NDetails::TStack AllocStack(const char* name);
- void FreeStack(NDetails::TStack& stack);
-
- uint64_t GetReleasedSize() const noexcept;
- uint64_t GetFullSize() const noexcept;
- uint64_t GetNumOfAllocated() const noexcept { return NumOfAllocated_; }
-
- private:
- void AllocNewMemoryChunk();
- bool IsSmallStack() const noexcept;
- bool IsStackFromThisPool(const NDetails::TStack& stack) const noexcept;
- NDetails::TStack AllocNewStack(const char* name);
-
- private:
- const uint64_t StackSize_ = 0;
- uint64_t RssPagesToKeep_ = 0;
- const TGuard& Guard_;
- TVector<TMemory> Memory_; // memory chunks
- THolder<TStorage> Storage_;
- char* NextToAlloc_ = nullptr; // points to next available stack in the last memory chunk
- const uint64_t ChunkSize_ = 0;
- uint64_t NumOfAllocated_ = 0;
- };
-
-}
-
-#include "stack_pool.inl"
+#pragma once
+
+#include "stack.h"
+#include "stack_common.h"
+
+#include <util/generic/noncopyable.h>
+#include <util/generic/ptr.h>
+#include <util/generic/vector.h>
+
+
+namespace NCoro::NStack {
+
+ class IGuard;
+ class TStorage;
+ struct TPoolAllocatorSettings;
+
+ template<typename TGuard>
+ class TPool final : private TMoveOnly {
+ struct TMemory {
+ char* Raw = nullptr;
+ char* Aligned = nullptr; // points to aligned memory, which includes space for first page guard
+ };
+ public:
+ TPool(uint64_t stackSize, const TPoolAllocatorSettings& settings, const TGuard& guard);
+ TPool(TPool&& other) noexcept;
+ ~TPool();
+
+ NDetails::TStack AllocStack(const char* name);
+ void FreeStack(NDetails::TStack& stack);
+
+ uint64_t GetReleasedSize() const noexcept;
+ uint64_t GetFullSize() const noexcept;
+ uint64_t GetNumOfAllocated() const noexcept { return NumOfAllocated_; }
+
+ private:
+ void AllocNewMemoryChunk();
+ bool IsSmallStack() const noexcept;
+ bool IsStackFromThisPool(const NDetails::TStack& stack) const noexcept;
+ NDetails::TStack AllocNewStack(const char* name);
+
+ private:
+ const uint64_t StackSize_ = 0;
+ uint64_t RssPagesToKeep_ = 0;
+ const TGuard& Guard_;
+ TVector<TMemory> Memory_; // memory chunks
+ THolder<TStorage> Storage_;
+ char* NextToAlloc_ = nullptr; // points to next available stack in the last memory chunk
+ const uint64_t ChunkSize_ = 0;
+ uint64_t NumOfAllocated_ = 0;
+ };
+
+}
+
+#include "stack_pool.inl"
diff --git a/library/cpp/coroutine/engine/stack/stack_pool.inl b/library/cpp/coroutine/engine/stack/stack_pool.inl
index 8a81514158..6e08e05a48 100644
--- a/library/cpp/coroutine/engine/stack/stack_pool.inl
+++ b/library/cpp/coroutine/engine/stack/stack_pool.inl
@@ -1,132 +1,132 @@
-#include "stack_storage.h"
-#include "stack_utils.h"
-
-
-namespace NCoro::NStack {
-
- template<typename TGuard>
- TPool<TGuard>::TPool(uint64_t stackSize, const TPoolAllocatorSettings& settings, const TGuard& guard)
- : StackSize_(stackSize)
- , RssPagesToKeep_(IsSmallStack() ? settings.SmallStackRssPagesToKeep : settings.RssPagesToKeep)
- , Guard_(guard)
- , ChunkSize_(Guard_.GetPageAlignedSize() + StackSize_ * settings.StacksPerChunk)
- {
- Y_ASSERT(RssPagesToKeep_);
- if (!RssPagesToKeep_) {
- RssPagesToKeep_ = 1; // at least guard should be kept
- }
-
- const uint64_t stackSizeInPages = stackSize / PageSize;
- Y_ASSERT(stackSizeInPages >= RssPagesToKeep_);
- if (stackSizeInPages < RssPagesToKeep_) {
- RssPagesToKeep_ = stackSizeInPages; // keep all stack pages
- }
-
- Y_ASSERT(StackSize_ && !(StackSize_ & PageSizeMask)); // stack size is not zero and page aligned
- Y_ASSERT(Guard_.GetSize() < StackSize_); // stack has enough space to place guard
- Y_ASSERT(stackSizeInPages >= RssPagesToKeep_);
-
- Storage_ = MakeHolder<TStorage>(StackSize_, RssPagesToKeep_, settings.ReleaseRate);
-
- AllocNewMemoryChunk();
- }
-
- template<typename TGuard>
- TPool<TGuard>::TPool(TPool&& other) noexcept = default;
-
- template<typename TGuard>
- TPool<TGuard>::~TPool() {
- if (!Memory_.empty()) {
- Y_ASSERT(NextToAlloc_ && StackSize_);
-
- for (const auto& chunk : Memory_) {
- Y_ASSERT(chunk.Raw && chunk.Aligned);
-
- if (Guard_.ShouldRemoveProtectionBeforeFree()) {
- Guard_.RemoveProtection(chunk.Aligned, Guard_.GetPageAlignedSize()); // first page in chunk
-
- const char* endOfStacksMemory = chunk.Aligned + ChunkSize_;
- for (char* i = chunk.Aligned + Guard_.GetPageAlignedSize(); i < endOfStacksMemory; i += StackSize_) {
- Guard_.RemoveProtection(i, StackSize_);
- }
- }
-
- free(chunk.Raw);
- }
- }
- }
-
- template<typename TGuard>
- NDetails::TStack TPool<TGuard>::AllocStack(const char* name) {
- Y_ASSERT(!Memory_.empty());
-
- if (!Storage_->IsEmpty()) {
- return Storage_->GetStack(Guard_, name);
- } else {
- ++NumOfAllocated_;
- return AllocNewStack(name);
- }
- }
-
- template<typename TGuard>
- void TPool<TGuard>::FreeStack(NDetails::TStack& stack) {
- Y_ASSERT(Storage_->Size() < ((ChunkSize_ - Guard_.GetPageAlignedSize()) / StackSize_) * Memory_.size());
- Y_ASSERT(IsStackFromThisPool(stack));
-
- Storage_->ReturnStack(stack);
- }
-
- template<typename TGuard>
- uint64_t TPool<TGuard>::GetReleasedSize() const noexcept {
- return Storage_->GetReleasedSize();
- }
- template<typename TGuard>
- uint64_t TPool<TGuard>::GetFullSize() const noexcept {
- return Storage_->GetFullSize();
- }
-
- template<typename TGuard>
- void TPool<TGuard>::AllocNewMemoryChunk() {
- const uint64_t totalSizeInPages = ChunkSize_ / PageSize;
-
- TMemory memory;
- const auto res = GetAlignedMemory(totalSizeInPages, memory.Raw, memory.Aligned);
- Y_VERIFY(res, "Failed to allocate memory for coro stack pool");
-
- NextToAlloc_ = memory.Aligned + Guard_.GetPageAlignedSize(); // skip first guard page
- Guard_.Protect(memory.Aligned, Guard_.GetPageAlignedSize(), false); // protect first guard page
-
- Memory_.push_back(std::move(memory));
- }
-
- template<typename TGuard>
- bool TPool<TGuard>::IsSmallStack() const noexcept {
- return StackSize_ / PageSize <= SmallStackMaxSizeInPages;
- }
-
- template<typename TGuard>
- bool TPool<TGuard>::IsStackFromThisPool(const NDetails::TStack& stack) const noexcept {
- for (const auto& chunk : Memory_) {
- const char* endOfStacksMemory = chunk.Aligned + ChunkSize_;
- if (chunk.Raw <= stack.GetRawMemory() && stack.GetRawMemory() < endOfStacksMemory) {
- return true;
- }
- }
- return false;
- }
-
- template<typename TGuard>
- NDetails::TStack TPool<TGuard>::AllocNewStack(const char* name) {
- if (NextToAlloc_ + StackSize_ > Memory_.rbegin()->Aligned + ChunkSize_) {
- AllocNewMemoryChunk(); // also sets NextToAlloc_ to first stack position in new allocated chunk of memory
- }
- Y_ASSERT(NextToAlloc_ + StackSize_ <= Memory_.rbegin()->Aligned + ChunkSize_);
-
- char* newStack = NextToAlloc_;
- NextToAlloc_ += StackSize_;
-
- Guard_.Protect(newStack, StackSize_, true);
- return NDetails::TStack{newStack, newStack, StackSize_, name};
- }
-
-}
+#include "stack_storage.h"
+#include "stack_utils.h"
+
+
+namespace NCoro::NStack {
+
+ template<typename TGuard>
+ TPool<TGuard>::TPool(uint64_t stackSize, const TPoolAllocatorSettings& settings, const TGuard& guard)
+ : StackSize_(stackSize)
+ , RssPagesToKeep_(IsSmallStack() ? settings.SmallStackRssPagesToKeep : settings.RssPagesToKeep)
+ , Guard_(guard)
+ , ChunkSize_(Guard_.GetPageAlignedSize() + StackSize_ * settings.StacksPerChunk)
+ {
+ Y_ASSERT(RssPagesToKeep_);
+ if (!RssPagesToKeep_) {
+ RssPagesToKeep_ = 1; // at least guard should be kept
+ }
+
+ const uint64_t stackSizeInPages = stackSize / PageSize;
+ Y_ASSERT(stackSizeInPages >= RssPagesToKeep_);
+ if (stackSizeInPages < RssPagesToKeep_) {
+ RssPagesToKeep_ = stackSizeInPages; // keep all stack pages
+ }
+
+ Y_ASSERT(StackSize_ && !(StackSize_ & PageSizeMask)); // stack size is not zero and page aligned
+ Y_ASSERT(Guard_.GetSize() < StackSize_); // stack has enough space to place guard
+ Y_ASSERT(stackSizeInPages >= RssPagesToKeep_);
+
+ Storage_ = MakeHolder<TStorage>(StackSize_, RssPagesToKeep_, settings.ReleaseRate);
+
+ AllocNewMemoryChunk();
+ }
+
+ template<typename TGuard>
+ TPool<TGuard>::TPool(TPool&& other) noexcept = default;
+
+ template<typename TGuard>
+ TPool<TGuard>::~TPool() {
+ if (!Memory_.empty()) {
+ Y_ASSERT(NextToAlloc_ && StackSize_);
+
+ for (const auto& chunk : Memory_) {
+ Y_ASSERT(chunk.Raw && chunk.Aligned);
+
+ if (Guard_.ShouldRemoveProtectionBeforeFree()) {
+ Guard_.RemoveProtection(chunk.Aligned, Guard_.GetPageAlignedSize()); // first page in chunk
+
+ const char* endOfStacksMemory = chunk.Aligned + ChunkSize_;
+ for (char* i = chunk.Aligned + Guard_.GetPageAlignedSize(); i < endOfStacksMemory; i += StackSize_) {
+ Guard_.RemoveProtection(i, StackSize_);
+ }
+ }
+
+ free(chunk.Raw);
+ }
+ }
+ }
+
+ template<typename TGuard>
+ NDetails::TStack TPool<TGuard>::AllocStack(const char* name) {
+ Y_ASSERT(!Memory_.empty());
+
+ if (!Storage_->IsEmpty()) {
+ return Storage_->GetStack(Guard_, name);
+ } else {
+ ++NumOfAllocated_;
+ return AllocNewStack(name);
+ }
+ }
+
+ template<typename TGuard>
+ void TPool<TGuard>::FreeStack(NDetails::TStack& stack) {
+ Y_ASSERT(Storage_->Size() < ((ChunkSize_ - Guard_.GetPageAlignedSize()) / StackSize_) * Memory_.size());
+ Y_ASSERT(IsStackFromThisPool(stack));
+
+ Storage_->ReturnStack(stack);
+ }
+
+ template<typename TGuard>
+ uint64_t TPool<TGuard>::GetReleasedSize() const noexcept {
+ return Storage_->GetReleasedSize();
+ }
+ template<typename TGuard>
+ uint64_t TPool<TGuard>::GetFullSize() const noexcept {
+ return Storage_->GetFullSize();
+ }
+
+ template<typename TGuard>
+ void TPool<TGuard>::AllocNewMemoryChunk() {
+ const uint64_t totalSizeInPages = ChunkSize_ / PageSize;
+
+ TMemory memory;
+ const auto res = GetAlignedMemory(totalSizeInPages, memory.Raw, memory.Aligned);
+ Y_VERIFY(res, "Failed to allocate memory for coro stack pool");
+
+ NextToAlloc_ = memory.Aligned + Guard_.GetPageAlignedSize(); // skip first guard page
+ Guard_.Protect(memory.Aligned, Guard_.GetPageAlignedSize(), false); // protect first guard page
+
+ Memory_.push_back(std::move(memory));
+ }
+
+ template<typename TGuard>
+ bool TPool<TGuard>::IsSmallStack() const noexcept {
+ return StackSize_ / PageSize <= SmallStackMaxSizeInPages;
+ }
+
+ template<typename TGuard>
+ bool TPool<TGuard>::IsStackFromThisPool(const NDetails::TStack& stack) const noexcept {
+ for (const auto& chunk : Memory_) {
+ const char* endOfStacksMemory = chunk.Aligned + ChunkSize_;
+ if (chunk.Raw <= stack.GetRawMemory() && stack.GetRawMemory() < endOfStacksMemory) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ template<typename TGuard>
+ NDetails::TStack TPool<TGuard>::AllocNewStack(const char* name) {
+ if (NextToAlloc_ + StackSize_ > Memory_.rbegin()->Aligned + ChunkSize_) {
+ AllocNewMemoryChunk(); // also sets NextToAlloc_ to first stack position in new allocated chunk of memory
+ }
+ Y_ASSERT(NextToAlloc_ + StackSize_ <= Memory_.rbegin()->Aligned + ChunkSize_);
+
+ char* newStack = NextToAlloc_;
+ NextToAlloc_ += StackSize_;
+
+ Guard_.Protect(newStack, StackSize_, true);
+ return NDetails::TStack{newStack, newStack, StackSize_, name};
+ }
+
+}
diff --git a/library/cpp/coroutine/engine/stack/stack_storage.cpp b/library/cpp/coroutine/engine/stack/stack_storage.cpp
index c86fb62329..6dc2b2d44b 100644
--- a/library/cpp/coroutine/engine/stack/stack_storage.cpp
+++ b/library/cpp/coroutine/engine/stack/stack_storage.cpp
@@ -1,46 +1,46 @@
-#include "stack_storage.h"
-
-#include "stack.h"
-#include "stack_utils.h"
-
-#include <library/cpp/coroutine/engine/impl.h>
-
-
-namespace NCoro::NStack {
-
- TStorage::TStorage(uint64_t stackSize, uint64_t rssPagesToKeep, uint64_t releaseRate)
- : StackSize_(stackSize)
- , RssPagesToKeep_(rssPagesToKeep)
- , ReleaseRate_(releaseRate ? releaseRate : 1)
- {
- Y_ASSERT(StackSize_ && RssPagesToKeep_);
- }
-
- bool TStorage::IsEmpty() const noexcept {
- return Released_.empty() && Full_.empty();
- }
-
- uint64_t TStorage::Size() const noexcept {
- return Released_.size() + Full_.size();
- }
-
- void TStorage::ReturnStack(NDetails::TStack& stack) {
- thread_local uint64_t i = 0;
- if (++i % ReleaseRate_ != 0) {
- Full_.push_back(stack.GetAlignedMemory());
- } else {
- ReleaseMemory(stack.GetAlignedMemory(), RssPagesToKeep_);
- Released_.push_back(stack.GetAlignedMemory());
- }
- stack.Reset();
- }
-
- void TStorage::ReleaseMemory([[maybe_unused]] char* alignedStackMemory, [[maybe_unused]] uint64_t pagesToKeep) noexcept {
-#if !defined(_san_enabled_) && defined(NDEBUG)
- uint64_t numOfPagesToFree = StackSize_ / PageSize;
- numOfPagesToFree -= pagesToKeep;
- ReleaseRss(alignedStackMemory, numOfPagesToFree);
-#endif
- }
-
-}
+#include "stack_storage.h"
+
+#include "stack.h"
+#include "stack_utils.h"
+
+#include <library/cpp/coroutine/engine/impl.h>
+
+
+namespace NCoro::NStack {
+
+ TStorage::TStorage(uint64_t stackSize, uint64_t rssPagesToKeep, uint64_t releaseRate)
+ : StackSize_(stackSize)
+ , RssPagesToKeep_(rssPagesToKeep)
+ , ReleaseRate_(releaseRate ? releaseRate : 1)
+ {
+ Y_ASSERT(StackSize_ && RssPagesToKeep_);
+ }
+
+ bool TStorage::IsEmpty() const noexcept {
+ return Released_.empty() && Full_.empty();
+ }
+
+ uint64_t TStorage::Size() const noexcept {
+ return Released_.size() + Full_.size();
+ }
+
+ void TStorage::ReturnStack(NDetails::TStack& stack) {
+ thread_local uint64_t i = 0;
+ if (++i % ReleaseRate_ != 0) {
+ Full_.push_back(stack.GetAlignedMemory());
+ } else {
+ ReleaseMemory(stack.GetAlignedMemory(), RssPagesToKeep_);
+ Released_.push_back(stack.GetAlignedMemory());
+ }
+ stack.Reset();
+ }
+
+ void TStorage::ReleaseMemory([[maybe_unused]] char* alignedStackMemory, [[maybe_unused]] uint64_t pagesToKeep) noexcept {
+#if !defined(_san_enabled_) && defined(NDEBUG)
+ uint64_t numOfPagesToFree = StackSize_ / PageSize;
+ numOfPagesToFree -= pagesToKeep;
+ ReleaseRss(alignedStackMemory, numOfPagesToFree);
+#endif
+ }
+
+}
diff --git a/library/cpp/coroutine/engine/stack/stack_storage.h b/library/cpp/coroutine/engine/stack/stack_storage.h
index f9c348547c..25fe2cfb17 100644
--- a/library/cpp/coroutine/engine/stack/stack_storage.h
+++ b/library/cpp/coroutine/engine/stack/stack_storage.h
@@ -1,60 +1,60 @@
-#pragma once
-
-#include "stack.h"
-
-#include <util/datetime/base.h>
-#include <util/generic/deque.h>
-
-
-class TCont;
-
-namespace NCoro::NStack {
-
- class IGuard;
-
- class TStorage final : private TMoveOnly {
- public:
- TStorage(uint64_t stackSize, uint64_t rssPagesToKeep, uint64_t releaseRate);
-
- bool IsEmpty() const noexcept;
- uint64_t Size() const noexcept;
-
- uint64_t GetReleasedSize() const noexcept { return Released_.size(); }
- uint64_t GetFullSize() const noexcept { return Full_.size(); }
-
- template<typename TGuard>
- NDetails::TStack GetStack(const TGuard& guard, const char* name);
- void ReturnStack(NDetails::TStack& stack);
-
- private:
- void ReleaseMemory(char* alignedStackMemory, uint64_t pagesToKeep) noexcept;
-
- private:
- TDeque<void*> Released_; //!< stacks memory with released RSS memory
- TDeque<void*> Full_; //!< stacks memory with RSS memory
- uint64_t StackSize_ = 0;
- uint64_t RssPagesToKeep_ = 0;
- const uint64_t ReleaseRate_ = 1;
- };
-
-
- template<typename TGuard>
- NDetails::TStack TStorage::GetStack(const TGuard& guard, const char* name) {
- Y_VERIFY(!IsEmpty()); // check before call
-
- void* newStack = nullptr;
- if (!Full_.empty()) {
- newStack = Full_.back();
- Full_.pop_back();
- } else {
- Y_ASSERT(!Released_.empty());
- newStack = Released_.back();
- Released_.pop_back();
- }
-
- Y_VERIFY(guard.CheckOverflow(newStack), "corrupted stack in pool");
- Y_VERIFY(guard.CheckOverride(newStack, StackSize_), "corrupted stack in pool");
-
- return NDetails::TStack{newStack, newStack, StackSize_, name};
- }
-}
+#pragma once
+
+#include "stack.h"
+
+#include <util/datetime/base.h>
+#include <util/generic/deque.h>
+
+
+class TCont;
+
+namespace NCoro::NStack {
+
+ class IGuard;
+
+ class TStorage final : private TMoveOnly {
+ public:
+ TStorage(uint64_t stackSize, uint64_t rssPagesToKeep, uint64_t releaseRate);
+
+ bool IsEmpty() const noexcept;
+ uint64_t Size() const noexcept;
+
+ uint64_t GetReleasedSize() const noexcept { return Released_.size(); }
+ uint64_t GetFullSize() const noexcept { return Full_.size(); }
+
+ template<typename TGuard>
+ NDetails::TStack GetStack(const TGuard& guard, const char* name);
+ void ReturnStack(NDetails::TStack& stack);
+
+ private:
+ void ReleaseMemory(char* alignedStackMemory, uint64_t pagesToKeep) noexcept;
+
+ private:
+ TDeque<void*> Released_; //!< stacks memory with released RSS memory
+ TDeque<void*> Full_; //!< stacks memory with RSS memory
+ uint64_t StackSize_ = 0;
+ uint64_t RssPagesToKeep_ = 0;
+ const uint64_t ReleaseRate_ = 1;
+ };
+
+
+ template<typename TGuard>
+ NDetails::TStack TStorage::GetStack(const TGuard& guard, const char* name) {
+ Y_VERIFY(!IsEmpty()); // check before call
+
+ void* newStack = nullptr;
+ if (!Full_.empty()) {
+ newStack = Full_.back();
+ Full_.pop_back();
+ } else {
+ Y_ASSERT(!Released_.empty());
+ newStack = Released_.back();
+ Released_.pop_back();
+ }
+
+ Y_VERIFY(guard.CheckOverflow(newStack), "corrupted stack in pool");
+ Y_VERIFY(guard.CheckOverride(newStack, StackSize_), "corrupted stack in pool");
+
+ return NDetails::TStack{newStack, newStack, StackSize_, name};
+ }
+}
diff --git a/library/cpp/coroutine/engine/stack/stack_utils.cpp b/library/cpp/coroutine/engine/stack/stack_utils.cpp
index ceeb3b50ec..1548529b66 100644
--- a/library/cpp/coroutine/engine/stack/stack_utils.cpp
+++ b/library/cpp/coroutine/engine/stack/stack_utils.cpp
@@ -1,84 +1,84 @@
-#include "stack_utils.h"
-
-#include <contrib/libs/linux-headers/asm-generic/errno-base.h>
-#include <util/generic/scope.h>
-#include <util/system/yassert.h>
-
-#ifdef _linux_
-#include <sys/mman.h>
-#endif
-
-#include <cerrno>
-#include <cstdlib>
-#include <cstring>
-
-
-namespace NCoro::NStack {
-
-#ifdef _linux_
- bool GetAlignedMemory(uint64_t sizeInPages, char*& rawPtr, char*& alignedPtr) noexcept {
- Y_ASSERT(sizeInPages);
-
- void* ptr = nullptr;
- int error = posix_memalign(&ptr, PageSize, sizeInPages * PageSize);
- alignedPtr = rawPtr = (char*)ptr;
- return rawPtr && alignedPtr && !error;
- }
-#else
- bool GetAlignedMemory(uint64_t sizeInPages, char*& rawPtr, char*& alignedPtr) noexcept {
- Y_ASSERT(sizeInPages);
-
- rawPtr = (char*) malloc((sizeInPages + 1) * PageSize); // +1 in case result would be unaligned
- alignedPtr = (char*)( ((uint64_t)rawPtr + PageSize - 1) & ~PageSizeMask);
- return rawPtr && alignedPtr;
- }
-#endif
-
-#ifdef _linux_
- void ReleaseRss(char* alignedPtr, uint64_t numOfPages) noexcept {
- Y_VERIFY( !((uint64_t)alignedPtr & PageSizeMask), "Not aligned pointer to release RSS memory");
- if (!numOfPages) {
- return;
- }
- if (auto res = madvise((void*) alignedPtr, numOfPages * PageSize, MADV_DONTNEED); res) {
- Y_VERIFY(errno == EAGAIN || errno == ENOMEM, "Failed to release memory");
- }
- }
-#else
- void ReleaseRss(char*, uint64_t) noexcept {
- }
-#endif
-
-#ifdef _linux_
- uint64_t CountMapped(char* alignedPtr, uint64_t numOfPages) noexcept {
- Y_VERIFY( !((uint64_t)alignedPtr & PageSizeMask) );
- Y_ASSERT(numOfPages);
-
- uint64_t result = 0;
- unsigned char* mappedPages = (unsigned char*) calloc(numOfPages, numOfPages);
- Y_VERIFY(mappedPages);
- Y_DEFER {
- free(mappedPages);
- };
-
- if (!mincore((void*)alignedPtr, numOfPages * PageSize, mappedPages)) {
- for (uint64_t i = 0; i < numOfPages; ++i) {
- if (mappedPages[i] & 1) {
- ++result;
- }
- }
- } else {
- Y_ASSERT(false);
- return 0;
- }
-
- return result;
- }
-
-#else
- uint64_t CountMapped(char*, uint64_t) noexcept {
- return 0; // stub for Windows tests
- }
-#endif
-
-}
+#include "stack_utils.h"
+
+#include <contrib/libs/linux-headers/asm-generic/errno-base.h>
+#include <util/generic/scope.h>
+#include <util/system/yassert.h>
+
+#ifdef _linux_
+#include <sys/mman.h>
+#endif
+
+#include <cerrno>
+#include <cstdlib>
+#include <cstring>
+
+
+namespace NCoro::NStack {
+
+#ifdef _linux_
+ bool GetAlignedMemory(uint64_t sizeInPages, char*& rawPtr, char*& alignedPtr) noexcept {
+ Y_ASSERT(sizeInPages);
+
+ void* ptr = nullptr;
+ int error = posix_memalign(&ptr, PageSize, sizeInPages * PageSize);
+ alignedPtr = rawPtr = (char*)ptr;
+ return rawPtr && alignedPtr && !error;
+ }
+#else
+ bool GetAlignedMemory(uint64_t sizeInPages, char*& rawPtr, char*& alignedPtr) noexcept {
+ Y_ASSERT(sizeInPages);
+
+ rawPtr = (char*) malloc((sizeInPages + 1) * PageSize); // +1 in case result would be unaligned
+ alignedPtr = (char*)( ((uint64_t)rawPtr + PageSize - 1) & ~PageSizeMask);
+ return rawPtr && alignedPtr;
+ }
+#endif
+
+#ifdef _linux_
+ void ReleaseRss(char* alignedPtr, uint64_t numOfPages) noexcept {
+ Y_VERIFY( !((uint64_t)alignedPtr & PageSizeMask), "Not aligned pointer to release RSS memory");
+ if (!numOfPages) {
+ return;
+ }
+ if (auto res = madvise((void*) alignedPtr, numOfPages * PageSize, MADV_DONTNEED); res) {
+ Y_VERIFY(errno == EAGAIN || errno == ENOMEM, "Failed to release memory");
+ }
+ }
+#else
+ void ReleaseRss(char*, uint64_t) noexcept {
+ }
+#endif
+
+#ifdef _linux_
+ uint64_t CountMapped(char* alignedPtr, uint64_t numOfPages) noexcept {
+ Y_VERIFY( !((uint64_t)alignedPtr & PageSizeMask) );
+ Y_ASSERT(numOfPages);
+
+ uint64_t result = 0;
+ unsigned char* mappedPages = (unsigned char*) calloc(numOfPages, numOfPages);
+ Y_VERIFY(mappedPages);
+ Y_DEFER {
+ free(mappedPages);
+ };
+
+ if (!mincore((void*)alignedPtr, numOfPages * PageSize, mappedPages)) {
+ for (uint64_t i = 0; i < numOfPages; ++i) {
+ if (mappedPages[i] & 1) {
+ ++result;
+ }
+ }
+ } else {
+ Y_ASSERT(false);
+ return 0;
+ }
+
+ return result;
+ }
+
+#else
+ uint64_t CountMapped(char*, uint64_t) noexcept {
+ return 0; // stub for Windows tests
+ }
+#endif
+
+}
diff --git a/library/cpp/coroutine/engine/stack/stack_utils.h b/library/cpp/coroutine/engine/stack/stack_utils.h
index ba25cc38dc..46c65240b5 100644
--- a/library/cpp/coroutine/engine/stack/stack_utils.h
+++ b/library/cpp/coroutine/engine/stack/stack_utils.h
@@ -1,27 +1,27 @@
-#pragma once
-
-#include "stack_common.h"
-
-
-namespace NCoro::NStack {
- /*! Actual size of allocated memory can exceed size in pages, due to unaligned allocation.
- * @param sizeInPages : number of pages to allocate
- * @param rawPtr : pointer to unaligned memory. Should be passed to free() when is not used any more.
- * @param alignedPtr : pointer to beginning of first fully allocated page
- * @return : true on success
- */
- bool GetAlignedMemory(uint64_t sizeInPages, char*& rawPtr, char*& alignedPtr) noexcept;
-
- /*! Release mapped RSS memory.
- * @param alignedPt : page-size aligned memory on which RSS memory should be freed
- * @param numOfPages : number of pages to free from RSS memory
- */
- void ReleaseRss(char* alignedPtr, uint64_t numOfPages) noexcept;
-
- /*! Count pages with RSS memory
- * @param alignedPtr : pointer to page-aligned memory for which calculations would be done
- * @param numOfPages : number of pages to check
- * @return : number of pages with RSS memory
- */
- uint64_t CountMapped(char* alignedPtr, uint64_t numOfPages) noexcept;
-}
+#pragma once
+
+#include "stack_common.h"
+
+
+namespace NCoro::NStack {
+ /*! Actual size of allocated memory can exceed size in pages, due to unaligned allocation.
+ * @param sizeInPages : number of pages to allocate
+ * @param rawPtr : pointer to unaligned memory. Should be passed to free() when is not used any more.
+ * @param alignedPtr : pointer to beginning of first fully allocated page
+ * @return : true on success
+ */
+ bool GetAlignedMemory(uint64_t sizeInPages, char*& rawPtr, char*& alignedPtr) noexcept;
+
+ /*! Release mapped RSS memory.
+ * @param alignedPt : page-size aligned memory on which RSS memory should be freed
+ * @param numOfPages : number of pages to free from RSS memory
+ */
+ void ReleaseRss(char* alignedPtr, uint64_t numOfPages) noexcept;
+
+ /*! Count pages with RSS memory
+ * @param alignedPtr : pointer to page-aligned memory for which calculations would be done
+ * @param numOfPages : number of pages to check
+ * @return : number of pages with RSS memory
+ */
+ uint64_t CountMapped(char* alignedPtr, uint64_t numOfPages) noexcept;
+}
diff --git a/library/cpp/coroutine/engine/stack/ut/stack_allocator_ut.cpp b/library/cpp/coroutine/engine/stack/ut/stack_allocator_ut.cpp
index 718dd2352e..a7283d44a3 100644
--- a/library/cpp/coroutine/engine/stack/ut/stack_allocator_ut.cpp
+++ b/library/cpp/coroutine/engine/stack/ut/stack_allocator_ut.cpp
@@ -1,115 +1,115 @@
-#include <library/cpp/coroutine/engine/stack/stack_allocator.h>
-#include <library/cpp/coroutine/engine/stack/stack_common.h>
-#include <library/cpp/testing/gtest/gtest.h>
-
-
-using namespace testing;
-
-namespace NCoro::NStack::Tests {
-
- enum class EAllocator {
- Pool, // allocates page-size aligned stacks from pools
- Simple // uses malloc/free for each stack
- };
-
- class TAllocatorParamFixture : public TestWithParam< std::tuple<EGuard, EAllocator> > {
- protected: // methods
- void SetUp() override {
- EGuard guardType;
- EAllocator allocType;
- std::tie(guardType, allocType) = GetParam();
-
- TMaybe<TPoolAllocatorSettings> poolSettings;
- if (allocType == EAllocator::Pool) {
- poolSettings = TPoolAllocatorSettings{};
- }
-
- Allocator_ = GetAllocator(poolSettings, guardType);
- }
-
- protected: // data
- THolder<IAllocator> Allocator_;
- };
-
-
- TEST_P(TAllocatorParamFixture, StackAllocationAndRelease) {
- uint64_t stackSize = PageSize * 12;
- auto stack = Allocator_->AllocStack(stackSize, "test_stack");
-#if defined(_san_enabled_) || !defined(NDEBUG)
- stackSize *= DebugOrSanStackMultiplier;
-#endif
-
- // Correct stack should have
- EXPECT_EQ(stack.GetSize(), stackSize); // predefined size
- EXPECT_FALSE((uint64_t)stack.GetAlignedMemory() & PageSizeMask); // aligned pointer
- // Writable workspace
- auto workspace = Allocator_->GetStackWorkspace(stack.GetAlignedMemory(), stack.GetSize());
- for (uint64_t i = 0; i < workspace.size(); i += 512) {
- workspace[i] = 42;
- }
- EXPECT_TRUE(Allocator_->CheckStackOverflow(stack.GetAlignedMemory()));
- EXPECT_TRUE(Allocator_->CheckStackOverride(stack.GetAlignedMemory(), stack.GetSize()));
-
- Allocator_->FreeStack(stack);
- EXPECT_FALSE(stack.GetRawMemory());
- }
-
- INSTANTIATE_TEST_SUITE_P(AllocatorTestParams, TAllocatorParamFixture,
- Combine(Values(EGuard::Canary, EGuard::Page), Values(EAllocator::Pool, EAllocator::Simple)));
-
-
- // ------------------------------------------------------------------------
- // Test that allocated stack has guards
- //
- template<class AllocatorType>
- THolder<IAllocator> GetAllocator(EGuard guardType);
-
- struct TPoolTag {};
- struct TSimpleTag {};
-
- template<>
- THolder<IAllocator> GetAllocator<TPoolTag>(EGuard guardType) {
- TMaybe<TPoolAllocatorSettings> poolSettings = TPoolAllocatorSettings{};
- return GetAllocator(poolSettings, guardType);
- }
-
- template<>
- THolder<IAllocator> GetAllocator<TSimpleTag>(EGuard guardType) {
- TMaybe<TPoolAllocatorSettings> poolSettings;
- return GetAllocator(poolSettings, guardType);
- }
-
-
- template <class AllocatorType>
- class TAllocatorFixture : public Test {
- protected:
- TAllocatorFixture()
- : Allocator_(GetAllocator<AllocatorType>(EGuard::Page))
- {}
-
- const uint64_t StackSize_ = PageSize * 2;
- THolder<IAllocator> Allocator_;
- };
-
- typedef Types<TPoolTag, TSimpleTag> Implementations;
- TYPED_TEST_SUITE(TAllocatorFixture, Implementations);
-
- TYPED_TEST(TAllocatorFixture, StackOverflow) {
- ASSERT_DEATH({
- auto stack = this->Allocator_->AllocStack(this->StackSize_, "test_stack");
-
- // Overwrite previous guard, crash is here
- *(stack.GetAlignedMemory() - 1) = 42;
- }, "");
- }
-
- TYPED_TEST(TAllocatorFixture, StackOverride) {
- ASSERT_DEATH({
- auto stack = this->Allocator_->AllocStack(this->StackSize_, "test_stack");
-
- // Overwrite guard, crash is here
- *(stack.GetAlignedMemory() + stack.GetSize() - 1) = 42;
- }, "");
- }
-
-}
+#include <library/cpp/coroutine/engine/stack/stack_allocator.h>
+#include <library/cpp/coroutine/engine/stack/stack_common.h>
+#include <library/cpp/testing/gtest/gtest.h>
+
+
+using namespace testing;
+
+namespace NCoro::NStack::Tests {
+
+ enum class EAllocator {
+ Pool, // allocates page-size aligned stacks from pools
+ Simple // uses malloc/free for each stack
+ };
+
+ class TAllocatorParamFixture : public TestWithParam< std::tuple<EGuard, EAllocator> > {
+ protected: // methods
+ void SetUp() override {
+ EGuard guardType;
+ EAllocator allocType;
+ std::tie(guardType, allocType) = GetParam();
+
+ TMaybe<TPoolAllocatorSettings> poolSettings;
+ if (allocType == EAllocator::Pool) {
+ poolSettings = TPoolAllocatorSettings{};
+ }
+
+ Allocator_ = GetAllocator(poolSettings, guardType);
+ }
+
+ protected: // data
+ THolder<IAllocator> Allocator_;
+ };
+
+
+ TEST_P(TAllocatorParamFixture, StackAllocationAndRelease) {
+ uint64_t stackSize = PageSize * 12;
+ auto stack = Allocator_->AllocStack(stackSize, "test_stack");
+#if defined(_san_enabled_) || !defined(NDEBUG)
+ stackSize *= DebugOrSanStackMultiplier;
+#endif
+
+ // Correct stack should have
+ EXPECT_EQ(stack.GetSize(), stackSize); // predefined size
+ EXPECT_FALSE((uint64_t)stack.GetAlignedMemory() & PageSizeMask); // aligned pointer
+ // Writable workspace
+ auto workspace = Allocator_->GetStackWorkspace(stack.GetAlignedMemory(), stack.GetSize());
+ for (uint64_t i = 0; i < workspace.size(); i += 512) {
+ workspace[i] = 42;
+ }
+ EXPECT_TRUE(Allocator_->CheckStackOverflow(stack.GetAlignedMemory()));
+ EXPECT_TRUE(Allocator_->CheckStackOverride(stack.GetAlignedMemory(), stack.GetSize()));
+
+ Allocator_->FreeStack(stack);
+ EXPECT_FALSE(stack.GetRawMemory());
+ }
+
+ INSTANTIATE_TEST_SUITE_P(AllocatorTestParams, TAllocatorParamFixture,
+ Combine(Values(EGuard::Canary, EGuard::Page), Values(EAllocator::Pool, EAllocator::Simple)));
+
+
+ // ------------------------------------------------------------------------
+ // Test that allocated stack has guards
+ //
+ template<class AllocatorType>
+ THolder<IAllocator> GetAllocator(EGuard guardType);
+
+ struct TPoolTag {};
+ struct TSimpleTag {};
+
+ template<>
+ THolder<IAllocator> GetAllocator<TPoolTag>(EGuard guardType) {
+ TMaybe<TPoolAllocatorSettings> poolSettings = TPoolAllocatorSettings{};
+ return GetAllocator(poolSettings, guardType);
+ }
+
+ template<>
+ THolder<IAllocator> GetAllocator<TSimpleTag>(EGuard guardType) {
+ TMaybe<TPoolAllocatorSettings> poolSettings;
+ return GetAllocator(poolSettings, guardType);
+ }
+
+
+ template <class AllocatorType>
+ class TAllocatorFixture : public Test {
+ protected:
+ TAllocatorFixture()
+ : Allocator_(GetAllocator<AllocatorType>(EGuard::Page))
+ {}
+
+ const uint64_t StackSize_ = PageSize * 2;
+ THolder<IAllocator> Allocator_;
+ };
+
+ typedef Types<TPoolTag, TSimpleTag> Implementations;
+ TYPED_TEST_SUITE(TAllocatorFixture, Implementations);
+
+ TYPED_TEST(TAllocatorFixture, StackOverflow) {
+ ASSERT_DEATH({
+ auto stack = this->Allocator_->AllocStack(this->StackSize_, "test_stack");
+
+ // Overwrite previous guard, crash is here
+ *(stack.GetAlignedMemory() - 1) = 42;
+ }, "");
+ }
+
+ TYPED_TEST(TAllocatorFixture, StackOverride) {
+ ASSERT_DEATH({
+ auto stack = this->Allocator_->AllocStack(this->StackSize_, "test_stack");
+
+ // Overwrite guard, crash is here
+ *(stack.GetAlignedMemory() + stack.GetSize() - 1) = 42;
+ }, "");
+ }
+
+}
diff --git a/library/cpp/coroutine/engine/stack/ut/stack_guards_ut.cpp b/library/cpp/coroutine/engine/stack/ut/stack_guards_ut.cpp
index 6b1a18d9ad..9da9a9b3d5 100644
--- a/library/cpp/coroutine/engine/stack/ut/stack_guards_ut.cpp
+++ b/library/cpp/coroutine/engine/stack/ut/stack_guards_ut.cpp
@@ -1,158 +1,158 @@
-#include <library/cpp/coroutine/engine/stack/stack_common.h>
-#include <library/cpp/coroutine/engine/stack/stack_guards.h>
-#include <library/cpp/coroutine/engine/stack/stack_utils.h>
-#include <library/cpp/testing/gtest/gtest.h>
-
-
-using namespace testing;
-
-namespace NCoro::NStack::Tests {
-
- template <class TGuard>
- class TGuardFixture : public Test {
- protected:
- TGuardFixture() : Guard_(GetGuard<TGuard>()) {}
-
- const TGuard& Guard_;
- };
-
- typedef Types<TCanaryGuard, TPageGuard> Implementations;
- TYPED_TEST_SUITE(TGuardFixture, Implementations);
-
- TYPED_TEST(TGuardFixture, GuardSize) {
- const auto size = this->Guard_.GetSize();
- EXPECT_GE(size, 64ul);
- EXPECT_FALSE(size & 63ul); // check 64-byte alignment
- }
-
- TYPED_TEST(TGuardFixture, GuardAlignedSize) {
- const auto size = this->Guard_.GetPageAlignedSize();
- EXPECT_GE(size, PageSize);
- EXPECT_FALSE(size & PageSizeMask); // check page-alignment
- }
-
- TYPED_TEST(TGuardFixture, StackWorkspace) {
- for (uint64_t sizeInPages : {2, 5, 12}) {
- char *rawPtr, *alignedPtr = nullptr;
- ASSERT_TRUE(GetAlignedMemory(sizeInPages, rawPtr, alignedPtr));
- auto workspace = this->Guard_.GetWorkspace(alignedPtr, sizeInPages * PageSize);
- EXPECT_EQ(workspace.size(), sizeInPages * PageSize - this->Guard_.GetSize()) << " size in pages " << sizeInPages;
-
- this->Guard_.Protect(alignedPtr, sizeInPages * PageSize, false);
- workspace = this->Guard_.GetWorkspace(alignedPtr, sizeInPages * PageSize);
- EXPECT_EQ(workspace.size(), sizeInPages * PageSize - this->Guard_.GetSize()) << " size in pages " << sizeInPages;
-
- this->Guard_.RemoveProtection(alignedPtr, sizeInPages * PageSize);
- workspace = this->Guard_.GetWorkspace(alignedPtr, sizeInPages * PageSize);
- EXPECT_EQ(workspace.size(), sizeInPages * PageSize - this->Guard_.GetSize()) << " size in pages " << sizeInPages;
-
- free(rawPtr);
- }
- }
-
- TYPED_TEST(TGuardFixture, SetRemoveProtectionWorks) {
- char *rawPtr, *alignedPtr = nullptr;
- constexpr uint64_t sizeInPages = 4;
- ASSERT_TRUE(GetAlignedMemory(sizeInPages + 1, rawPtr, alignedPtr));
-
- this->Guard_.Protect(alignedPtr, PageSize, false); // set previous guard
- alignedPtr += PageSize; // leave first page for previous guard
- this->Guard_.Protect(alignedPtr, sizeInPages * PageSize, true);
-
- EXPECT_TRUE(this->Guard_.CheckOverflow(alignedPtr));
- EXPECT_TRUE(this->Guard_.CheckOverride(alignedPtr, sizeInPages * PageSize));
-
- this->Guard_.RemoveProtection(alignedPtr, sizeInPages * PageSize);
- this->Guard_.RemoveProtection(alignedPtr - PageSize, PageSize); // remove previous guard
-
- free(rawPtr);
- }
-
- TEST(StackGuardTest, CanaryGuardTestOverflow) {
- const auto& guard = GetGuard<TCanaryGuard>();
-
- char *rawPtr, *alignedPtr = nullptr;
- constexpr uint64_t sizeInPages = 4;
- ASSERT_TRUE(GetAlignedMemory(sizeInPages + 1, rawPtr, alignedPtr));
- guard.Protect(alignedPtr, PageSize, false); // set previous guard
- alignedPtr += PageSize; // leave first page for previous guard
- guard.Protect(alignedPtr, sizeInPages * PageSize, true);
-
- EXPECT_TRUE(guard.CheckOverflow(alignedPtr));
- EXPECT_TRUE(guard.CheckOverride(alignedPtr, sizeInPages * PageSize));
-
- // Overwrite previous guard
- *(alignedPtr - 1) = 42;
-
- EXPECT_FALSE(guard.CheckOverflow(alignedPtr));
-
- free(rawPtr);
- }
-
- TEST(StackGuardTest, CanaryGuardTestOverride) {
- const auto& guard = GetGuard<TCanaryGuard>();
-
- char *rawPtr, *alignedPtr = nullptr;
- constexpr uint64_t sizeInPages = 4;
- ASSERT_TRUE(GetAlignedMemory(sizeInPages + 1, rawPtr, alignedPtr));
- guard.Protect(alignedPtr, PageSize, false); // set previous guard
- alignedPtr += PageSize; // leave first page for previous guard
- guard.Protect(alignedPtr, sizeInPages * PageSize, true);
-
- EXPECT_TRUE(guard.CheckOverflow(alignedPtr));
- EXPECT_TRUE(guard.CheckOverride(alignedPtr, sizeInPages * PageSize));
-
- // Overwrite guard
- *(alignedPtr + sizeInPages * PageSize - 1) = 42;
-
- EXPECT_FALSE(guard.CheckOverride(alignedPtr, sizeInPages * PageSize));
-
- free(rawPtr);
- }
-
- TEST(StackGuardDeathTest, PageGuardTestOverflow) {
- ASSERT_DEATH({
- const auto &guard = GetGuard<TPageGuard>();
-
- char* rawPtr = nullptr;
- char* alignedPtr = nullptr;
- constexpr uint64_t sizeInPages = 4;
- ASSERT_TRUE(GetAlignedMemory(sizeInPages + 1, rawPtr, alignedPtr));
-
- guard.Protect(alignedPtr, PageSize, false); // set previous guard
- alignedPtr += PageSize; // leave first page for previous guard
- guard.Protect(alignedPtr, sizeInPages * PageSize, true);
-
- // Overwrite previous guard, crash is here
- *(alignedPtr - 1) = 42;
-
- guard.RemoveProtection(alignedPtr, sizeInPages * PageSize);
- guard.RemoveProtection(alignedPtr - PageSize, PageSize); // remove previous guard
-
- free(rawPtr);
- }, "");
- }
-
- TEST(StackGuardDeathTest, PageGuardTestOverride) {
- ASSERT_DEATH({
- const auto &guard = GetGuard<TPageGuard>();
-
- char* rawPtr = nullptr;
- char* alignedPtr = nullptr;
- constexpr uint64_t sizeInPages = 4;
- ASSERT_TRUE(GetAlignedMemory(sizeInPages + 1, rawPtr, alignedPtr));
- guard.Protect(alignedPtr, PageSize, false); // set previous guard
- alignedPtr += PageSize; // leave first page for previous guard
- guard.Protect(alignedPtr, sizeInPages * PageSize, true);
-
- // Overwrite guard, crash is here
- *(alignedPtr + sizeInPages * PageSize - 1) = 42;
-
- guard.RemoveProtection(alignedPtr, sizeInPages * PageSize);
- guard.RemoveProtection(alignedPtr - PageSize, PageSize); // remove previous guard
-
- free(rawPtr);
- }, "");
- }
-
-}
+#include <library/cpp/coroutine/engine/stack/stack_common.h>
+#include <library/cpp/coroutine/engine/stack/stack_guards.h>
+#include <library/cpp/coroutine/engine/stack/stack_utils.h>
+#include <library/cpp/testing/gtest/gtest.h>
+
+
+using namespace testing;
+
+namespace NCoro::NStack::Tests {
+
+ template <class TGuard>
+ class TGuardFixture : public Test {
+ protected:
+ TGuardFixture() : Guard_(GetGuard<TGuard>()) {}
+
+ const TGuard& Guard_;
+ };
+
+ typedef Types<TCanaryGuard, TPageGuard> Implementations;
+ TYPED_TEST_SUITE(TGuardFixture, Implementations);
+
+ TYPED_TEST(TGuardFixture, GuardSize) {
+ const auto size = this->Guard_.GetSize();
+ EXPECT_GE(size, 64ul);
+ EXPECT_FALSE(size & 63ul); // check 64-byte alignment
+ }
+
+ TYPED_TEST(TGuardFixture, GuardAlignedSize) {
+ const auto size = this->Guard_.GetPageAlignedSize();
+ EXPECT_GE(size, PageSize);
+ EXPECT_FALSE(size & PageSizeMask); // check page-alignment
+ }
+
+ TYPED_TEST(TGuardFixture, StackWorkspace) {
+ for (uint64_t sizeInPages : {2, 5, 12}) {
+ char *rawPtr, *alignedPtr = nullptr;
+ ASSERT_TRUE(GetAlignedMemory(sizeInPages, rawPtr, alignedPtr));
+ auto workspace = this->Guard_.GetWorkspace(alignedPtr, sizeInPages * PageSize);
+ EXPECT_EQ(workspace.size(), sizeInPages * PageSize - this->Guard_.GetSize()) << " size in pages " << sizeInPages;
+
+ this->Guard_.Protect(alignedPtr, sizeInPages * PageSize, false);
+ workspace = this->Guard_.GetWorkspace(alignedPtr, sizeInPages * PageSize);
+ EXPECT_EQ(workspace.size(), sizeInPages * PageSize - this->Guard_.GetSize()) << " size in pages " << sizeInPages;
+
+ this->Guard_.RemoveProtection(alignedPtr, sizeInPages * PageSize);
+ workspace = this->Guard_.GetWorkspace(alignedPtr, sizeInPages * PageSize);
+ EXPECT_EQ(workspace.size(), sizeInPages * PageSize - this->Guard_.GetSize()) << " size in pages " << sizeInPages;
+
+ free(rawPtr);
+ }
+ }
+
+ TYPED_TEST(TGuardFixture, SetRemoveProtectionWorks) {
+ char *rawPtr, *alignedPtr = nullptr;
+ constexpr uint64_t sizeInPages = 4;
+ ASSERT_TRUE(GetAlignedMemory(sizeInPages + 1, rawPtr, alignedPtr));
+
+ this->Guard_.Protect(alignedPtr, PageSize, false); // set previous guard
+ alignedPtr += PageSize; // leave first page for previous guard
+ this->Guard_.Protect(alignedPtr, sizeInPages * PageSize, true);
+
+ EXPECT_TRUE(this->Guard_.CheckOverflow(alignedPtr));
+ EXPECT_TRUE(this->Guard_.CheckOverride(alignedPtr, sizeInPages * PageSize));
+
+ this->Guard_.RemoveProtection(alignedPtr, sizeInPages * PageSize);
+ this->Guard_.RemoveProtection(alignedPtr - PageSize, PageSize); // remove previous guard
+
+ free(rawPtr);
+ }
+
+ TEST(StackGuardTest, CanaryGuardTestOverflow) {
+ const auto& guard = GetGuard<TCanaryGuard>();
+
+ char *rawPtr, *alignedPtr = nullptr;
+ constexpr uint64_t sizeInPages = 4;
+ ASSERT_TRUE(GetAlignedMemory(sizeInPages + 1, rawPtr, alignedPtr));
+ guard.Protect(alignedPtr, PageSize, false); // set previous guard
+ alignedPtr += PageSize; // leave first page for previous guard
+ guard.Protect(alignedPtr, sizeInPages * PageSize, true);
+
+ EXPECT_TRUE(guard.CheckOverflow(alignedPtr));
+ EXPECT_TRUE(guard.CheckOverride(alignedPtr, sizeInPages * PageSize));
+
+ // Overwrite previous guard
+ *(alignedPtr - 1) = 42;
+
+ EXPECT_FALSE(guard.CheckOverflow(alignedPtr));
+
+ free(rawPtr);
+ }
+
+ TEST(StackGuardTest, CanaryGuardTestOverride) {
+ const auto& guard = GetGuard<TCanaryGuard>();
+
+ char *rawPtr, *alignedPtr = nullptr;
+ constexpr uint64_t sizeInPages = 4;
+ ASSERT_TRUE(GetAlignedMemory(sizeInPages + 1, rawPtr, alignedPtr));
+ guard.Protect(alignedPtr, PageSize, false); // set previous guard
+ alignedPtr += PageSize; // leave first page for previous guard
+ guard.Protect(alignedPtr, sizeInPages * PageSize, true);
+
+ EXPECT_TRUE(guard.CheckOverflow(alignedPtr));
+ EXPECT_TRUE(guard.CheckOverride(alignedPtr, sizeInPages * PageSize));
+
+ // Overwrite guard
+ *(alignedPtr + sizeInPages * PageSize - 1) = 42;
+
+ EXPECT_FALSE(guard.CheckOverride(alignedPtr, sizeInPages * PageSize));
+
+ free(rawPtr);
+ }
+
+ TEST(StackGuardDeathTest, PageGuardTestOverflow) {
+ ASSERT_DEATH({
+ const auto &guard = GetGuard<TPageGuard>();
+
+ char* rawPtr = nullptr;
+ char* alignedPtr = nullptr;
+ constexpr uint64_t sizeInPages = 4;
+ ASSERT_TRUE(GetAlignedMemory(sizeInPages + 1, rawPtr, alignedPtr));
+
+ guard.Protect(alignedPtr, PageSize, false); // set previous guard
+ alignedPtr += PageSize; // leave first page for previous guard
+ guard.Protect(alignedPtr, sizeInPages * PageSize, true);
+
+ // Overwrite previous guard, crash is here
+ *(alignedPtr - 1) = 42;
+
+ guard.RemoveProtection(alignedPtr, sizeInPages * PageSize);
+ guard.RemoveProtection(alignedPtr - PageSize, PageSize); // remove previous guard
+
+ free(rawPtr);
+ }, "");
+ }
+
+ TEST(StackGuardDeathTest, PageGuardTestOverride) {
+ ASSERT_DEATH({
+ const auto &guard = GetGuard<TPageGuard>();
+
+ char* rawPtr = nullptr;
+ char* alignedPtr = nullptr;
+ constexpr uint64_t sizeInPages = 4;
+ ASSERT_TRUE(GetAlignedMemory(sizeInPages + 1, rawPtr, alignedPtr));
+ guard.Protect(alignedPtr, PageSize, false); // set previous guard
+ alignedPtr += PageSize; // leave first page for previous guard
+ guard.Protect(alignedPtr, sizeInPages * PageSize, true);
+
+ // Overwrite guard, crash is here
+ *(alignedPtr + sizeInPages * PageSize - 1) = 42;
+
+ guard.RemoveProtection(alignedPtr, sizeInPages * PageSize);
+ guard.RemoveProtection(alignedPtr - PageSize, PageSize); // remove previous guard
+
+ free(rawPtr);
+ }, "");
+ }
+
+}
diff --git a/library/cpp/coroutine/engine/stack/ut/stack_pool_ut.cpp b/library/cpp/coroutine/engine/stack/ut/stack_pool_ut.cpp
index 6a2f730c03..9e3e5e7117 100644
--- a/library/cpp/coroutine/engine/stack/ut/stack_pool_ut.cpp
+++ b/library/cpp/coroutine/engine/stack/ut/stack_pool_ut.cpp
@@ -1,70 +1,70 @@
-#include <library/cpp/coroutine/engine/stack/stack_common.h>
-#include <library/cpp/coroutine/engine/stack/stack_guards.h>
-#include <library/cpp/coroutine/engine/stack/stack_pool.h>
-#include <library/cpp/testing/gtest/gtest.h>
-
-
-using namespace testing;
-
-namespace NCoro::NStack::Tests {
-
- template <class TGuard>
- class TPoolFixture : public Test {
- protected:
- TPoolFixture() : Guard_(GetGuard<TGuard>()), Pool_(StackSize_, TPoolAllocatorSettings{1, 1, 8, 32}, Guard_) {}
-
- const uint64_t StackSize_ = PageSize * 4;
- const TGuard& Guard_;
- TPool<TGuard> Pool_;
- };
-
- typedef Types<TCanaryGuard, TPageGuard> Implementations;
- TYPED_TEST_SUITE(TPoolFixture, Implementations);
-
- TYPED_TEST(TPoolFixture, AllocAndFreeStack) {
- auto stack = this->Pool_.AllocStack("test_stack");
- this->Pool_.FreeStack(stack);
- EXPECT_FALSE(stack.GetRawMemory());
- }
-
- TYPED_TEST(TPoolFixture, FreedStackReused) {
- auto stack = this->Pool_.AllocStack("test_stack");
- auto rawPtr = stack.GetRawMemory();
- auto alignedPtr = stack.GetAlignedMemory();
-
- this->Pool_.FreeStack(stack);
- EXPECT_FALSE(stack.GetRawMemory());
-
- auto stack2 = this->Pool_.AllocStack("test_stack");
- EXPECT_EQ(rawPtr, stack2.GetRawMemory());
- EXPECT_EQ(alignedPtr, stack2.GetAlignedMemory());
-
- this->Pool_.FreeStack(stack2);
- EXPECT_FALSE(stack2.GetRawMemory());
- }
-
- TYPED_TEST(TPoolFixture, MruFreedStackReused) {
- auto stack = this->Pool_.AllocStack("test_stack");
- auto rawPtr = stack.GetRawMemory();
- auto alignedPtr = stack.GetAlignedMemory();
- auto stack2 = this->Pool_.AllocStack("test_stack");
- auto stack3 = this->Pool_.AllocStack("test_stack");
-
- this->Pool_.FreeStack(stack2);
- EXPECT_FALSE(stack2.GetRawMemory());
-
- this->Pool_.FreeStack(stack);
- EXPECT_FALSE(stack.GetRawMemory());
-
- auto stack4 = this->Pool_.AllocStack("test_stack");
- EXPECT_EQ(rawPtr, stack4.GetRawMemory());
- EXPECT_EQ(alignedPtr, stack4.GetAlignedMemory());
-
- this->Pool_.FreeStack(stack3);
- EXPECT_FALSE(stack.GetRawMemory());
-
- this->Pool_.FreeStack(stack4);
- EXPECT_FALSE(stack4.GetRawMemory());
- }
-
-}
+#include <library/cpp/coroutine/engine/stack/stack_common.h>
+#include <library/cpp/coroutine/engine/stack/stack_guards.h>
+#include <library/cpp/coroutine/engine/stack/stack_pool.h>
+#include <library/cpp/testing/gtest/gtest.h>
+
+
+using namespace testing;
+
+namespace NCoro::NStack::Tests {
+
+ template <class TGuard>
+ class TPoolFixture : public Test {
+ protected:
+ TPoolFixture() : Guard_(GetGuard<TGuard>()), Pool_(StackSize_, TPoolAllocatorSettings{1, 1, 8, 32}, Guard_) {}
+
+ const uint64_t StackSize_ = PageSize * 4;
+ const TGuard& Guard_;
+ TPool<TGuard> Pool_;
+ };
+
+ typedef Types<TCanaryGuard, TPageGuard> Implementations;
+ TYPED_TEST_SUITE(TPoolFixture, Implementations);
+
+ TYPED_TEST(TPoolFixture, AllocAndFreeStack) {
+ auto stack = this->Pool_.AllocStack("test_stack");
+ this->Pool_.FreeStack(stack);
+ EXPECT_FALSE(stack.GetRawMemory());
+ }
+
+ TYPED_TEST(TPoolFixture, FreedStackReused) {
+ auto stack = this->Pool_.AllocStack("test_stack");
+ auto rawPtr = stack.GetRawMemory();
+ auto alignedPtr = stack.GetAlignedMemory();
+
+ this->Pool_.FreeStack(stack);
+ EXPECT_FALSE(stack.GetRawMemory());
+
+ auto stack2 = this->Pool_.AllocStack("test_stack");
+ EXPECT_EQ(rawPtr, stack2.GetRawMemory());
+ EXPECT_EQ(alignedPtr, stack2.GetAlignedMemory());
+
+ this->Pool_.FreeStack(stack2);
+ EXPECT_FALSE(stack2.GetRawMemory());
+ }
+
+ TYPED_TEST(TPoolFixture, MruFreedStackReused) {
+ auto stack = this->Pool_.AllocStack("test_stack");
+ auto rawPtr = stack.GetRawMemory();
+ auto alignedPtr = stack.GetAlignedMemory();
+ auto stack2 = this->Pool_.AllocStack("test_stack");
+ auto stack3 = this->Pool_.AllocStack("test_stack");
+
+ this->Pool_.FreeStack(stack2);
+ EXPECT_FALSE(stack2.GetRawMemory());
+
+ this->Pool_.FreeStack(stack);
+ EXPECT_FALSE(stack.GetRawMemory());
+
+ auto stack4 = this->Pool_.AllocStack("test_stack");
+ EXPECT_EQ(rawPtr, stack4.GetRawMemory());
+ EXPECT_EQ(alignedPtr, stack4.GetAlignedMemory());
+
+ this->Pool_.FreeStack(stack3);
+ EXPECT_FALSE(stack.GetRawMemory());
+
+ this->Pool_.FreeStack(stack4);
+ EXPECT_FALSE(stack4.GetRawMemory());
+ }
+
+}
diff --git a/library/cpp/coroutine/engine/stack/ut/stack_ut.cpp b/library/cpp/coroutine/engine/stack/ut/stack_ut.cpp
index a41fc6c940..31f8ad6b61 100644
--- a/library/cpp/coroutine/engine/stack/ut/stack_ut.cpp
+++ b/library/cpp/coroutine/engine/stack/ut/stack_ut.cpp
@@ -1,60 +1,60 @@
-#include <library/cpp/coroutine/engine/stack/stack.h>
-#include <library/cpp/coroutine/engine/stack/stack_common.h>
-#include <library/cpp/coroutine/engine/stack/stack_guards.h>
-#include <library/cpp/coroutine/engine/stack/stack_utils.h>
-#include <library/cpp/testing/gtest/gtest.h>
-
-
-using namespace testing;
-
-namespace NCoro::NStack::Tests {
-
- constexpr uint64_t StackSizeInPages = 4;
-
- template <class TGuard>
- class TStackFixture : public Test {
- protected: // methods
- TStackFixture()
- : Guard_(GetGuard<TGuard>())
- , StackSize_(StackSizeInPages * PageSize)
- {}
-
- void SetUp() override {
- ASSERT_TRUE(GetAlignedMemory(StackSizeInPages, RawMemory_, AlignedMemory_));
- Stack_ = MakeHolder<NDetails::TStack>(RawMemory_, AlignedMemory_, StackSize_, "test_stack");
- Guard_.Protect(AlignedMemory_, StackSize_, false);
- }
-
- void TearDown() override {
- Guard_.RemoveProtection(AlignedMemory_, StackSize_);
- free(Stack_->GetRawMemory());
- Stack_->Reset();
- EXPECT_EQ(Stack_->GetRawMemory(), nullptr);
- }
-
- protected: // data
- const TGuard& Guard_;
- const uint64_t StackSize_ = 0;
- char* RawMemory_ = nullptr;
- char* AlignedMemory_ = nullptr;
- THolder<NDetails::TStack> Stack_;
- };
-
- typedef Types<TCanaryGuard, TPageGuard> Implementations;
- TYPED_TEST_SUITE(TStackFixture, Implementations);
-
- TYPED_TEST(TStackFixture, PointersAndSize) {
- EXPECT_EQ(this->Stack_->GetRawMemory(), this->RawMemory_);
- EXPECT_EQ(this->Stack_->GetAlignedMemory(), this->AlignedMemory_);
- EXPECT_EQ(this->Stack_->GetSize(), this->StackSize_);
- }
-
- TYPED_TEST(TStackFixture, WriteStack) {
- auto workspace = this->Guard_.GetWorkspace(this->Stack_->GetAlignedMemory(), this->Stack_->GetSize());
- for (uint64_t i = 0; i < workspace.size(); i += 512) {
- workspace[i] = 42;
- }
- EXPECT_TRUE(this->Guard_.CheckOverride(this->Stack_->GetAlignedMemory(), this->Stack_->GetSize()));
- }
-
-}
+#include <library/cpp/coroutine/engine/stack/stack.h>
+#include <library/cpp/coroutine/engine/stack/stack_common.h>
+#include <library/cpp/coroutine/engine/stack/stack_guards.h>
+#include <library/cpp/coroutine/engine/stack/stack_utils.h>
+#include <library/cpp/testing/gtest/gtest.h>
+
+
+using namespace testing;
+
+namespace NCoro::NStack::Tests {
+
+ constexpr uint64_t StackSizeInPages = 4;
+
+ template <class TGuard>
+ class TStackFixture : public Test {
+ protected: // methods
+ TStackFixture()
+ : Guard_(GetGuard<TGuard>())
+ , StackSize_(StackSizeInPages * PageSize)
+ {}
+
+ void SetUp() override {
+ ASSERT_TRUE(GetAlignedMemory(StackSizeInPages, RawMemory_, AlignedMemory_));
+ Stack_ = MakeHolder<NDetails::TStack>(RawMemory_, AlignedMemory_, StackSize_, "test_stack");
+ Guard_.Protect(AlignedMemory_, StackSize_, false);
+ }
+
+ void TearDown() override {
+ Guard_.RemoveProtection(AlignedMemory_, StackSize_);
+ free(Stack_->GetRawMemory());
+ Stack_->Reset();
+ EXPECT_EQ(Stack_->GetRawMemory(), nullptr);
+ }
+
+ protected: // data
+ const TGuard& Guard_;
+ const uint64_t StackSize_ = 0;
+ char* RawMemory_ = nullptr;
+ char* AlignedMemory_ = nullptr;
+ THolder<NDetails::TStack> Stack_;
+ };
+
+ typedef Types<TCanaryGuard, TPageGuard> Implementations;
+ TYPED_TEST_SUITE(TStackFixture, Implementations);
+
+ TYPED_TEST(TStackFixture, PointersAndSize) {
+ EXPECT_EQ(this->Stack_->GetRawMemory(), this->RawMemory_);
+ EXPECT_EQ(this->Stack_->GetAlignedMemory(), this->AlignedMemory_);
+ EXPECT_EQ(this->Stack_->GetSize(), this->StackSize_);
+ }
+
+ TYPED_TEST(TStackFixture, WriteStack) {
+ auto workspace = this->Guard_.GetWorkspace(this->Stack_->GetAlignedMemory(), this->Stack_->GetSize());
+ for (uint64_t i = 0; i < workspace.size(); i += 512) {
+ workspace[i] = 42;
+ }
+ EXPECT_TRUE(this->Guard_.CheckOverride(this->Stack_->GetAlignedMemory(), this->Stack_->GetSize()));
+ }
+
+}
diff --git a/library/cpp/coroutine/engine/stack/ut/stack_utils_ut.cpp b/library/cpp/coroutine/engine/stack/ut/stack_utils_ut.cpp
index 90e5d94b68..dc0593dcf2 100644
--- a/library/cpp/coroutine/engine/stack/ut/stack_utils_ut.cpp
+++ b/library/cpp/coroutine/engine/stack/ut/stack_utils_ut.cpp
@@ -1,73 +1,73 @@
-#include <library/cpp/coroutine/engine/stack/stack_common.h>
-#include <library/cpp/coroutine/engine/stack/stack_utils.h>
-#include <library/cpp/testing/gtest/gtest.h>
-
-
-using namespace testing;
-
-namespace NCoro::NStack::Tests {
-
- TEST(StackUtilsTest, Allocation) {
- char *rawPtr, *alignedPtr = nullptr;
- for (uint64_t i : {1, 2, 3, 4, 11}) {
- EXPECT_TRUE(GetAlignedMemory(i, rawPtr, alignedPtr));
- EXPECT_TRUE(rawPtr);
- EXPECT_TRUE(alignedPtr);
- EXPECT_FALSE((uint64_t)alignedPtr & PageSizeMask);
- free(rawPtr);
- }
- }
-
-#if !defined(_san_enabled_) && defined(_linux_)
-
- TEST(StackUtilsTest, RssReleaseOnePage) {
- char *rawPtr, *alignedPtr = nullptr;
- for (uint64_t i : {1, 2, 8}) {
- EXPECT_TRUE(GetAlignedMemory(i, rawPtr, alignedPtr));
- EXPECT_TRUE(rawPtr);
- EXPECT_TRUE(alignedPtr);
- EXPECT_FALSE((uint64_t)alignedPtr & PageSizeMask);
-
- ReleaseRss(alignedPtr, i); // allocator can provide reused memory with RSS memory on it
- EXPECT_EQ(CountMapped(alignedPtr, i), 0ul); // no RSS memory allocated
-
- *(alignedPtr + (i - 1) * PageSize) = 42; // map RSS memory
- EXPECT_EQ(CountMapped(alignedPtr, i), 1ul);
-
- ReleaseRss(alignedPtr, i);
- EXPECT_EQ(CountMapped(alignedPtr, i), 0ul) << "number of pages " << i; // no RSS memory allocated
-
- free(rawPtr);
- }
- }
-
- TEST(StackUtilsTest, RssReleaseSeveralPages) {
- char *rawPtr, *alignedPtr = nullptr;
-
- for (uint64_t i : {1, 2, 5, 8}) {
- EXPECT_TRUE(GetAlignedMemory(i, rawPtr, alignedPtr));
- EXPECT_TRUE(rawPtr);
- EXPECT_TRUE(alignedPtr);
- EXPECT_FALSE((uint64_t)alignedPtr & PageSizeMask);
-
- ReleaseRss(alignedPtr, i); // allocator can provide reused memory with RSS memory on it
- EXPECT_EQ(CountMapped(alignedPtr, i), 0ul); // no RSS memory allocated
-
- for (uint64_t page = 0; page < i; ++page) {
- *(alignedPtr + page * PageSize) = 42; // map RSS memory
- EXPECT_EQ(CountMapped(alignedPtr, page + 1), page + 1);
- }
-
- const uint64_t pagesToKeep = (i > 2) ? 2 : i;
-
- ReleaseRss(alignedPtr, i - pagesToKeep);
- EXPECT_EQ(CountMapped(alignedPtr, i), pagesToKeep) << "number of pages " << i; // no RSS memory allocated
-
- free(rawPtr);
- }
- }
-
-#endif
-
-}
-
+#include <library/cpp/coroutine/engine/stack/stack_common.h>
+#include <library/cpp/coroutine/engine/stack/stack_utils.h>
+#include <library/cpp/testing/gtest/gtest.h>
+
+
+using namespace testing;
+
+namespace NCoro::NStack::Tests {
+
+ TEST(StackUtilsTest, Allocation) {
+ char *rawPtr, *alignedPtr = nullptr;
+ for (uint64_t i : {1, 2, 3, 4, 11}) {
+ EXPECT_TRUE(GetAlignedMemory(i, rawPtr, alignedPtr));
+ EXPECT_TRUE(rawPtr);
+ EXPECT_TRUE(alignedPtr);
+ EXPECT_FALSE((uint64_t)alignedPtr & PageSizeMask);
+ free(rawPtr);
+ }
+ }
+
+#if !defined(_san_enabled_) && defined(_linux_)
+
+ TEST(StackUtilsTest, RssReleaseOnePage) {
+ char *rawPtr, *alignedPtr = nullptr;
+ for (uint64_t i : {1, 2, 8}) {
+ EXPECT_TRUE(GetAlignedMemory(i, rawPtr, alignedPtr));
+ EXPECT_TRUE(rawPtr);
+ EXPECT_TRUE(alignedPtr);
+ EXPECT_FALSE((uint64_t)alignedPtr & PageSizeMask);
+
+ ReleaseRss(alignedPtr, i); // allocator can provide reused memory with RSS memory on it
+ EXPECT_EQ(CountMapped(alignedPtr, i), 0ul); // no RSS memory allocated
+
+ *(alignedPtr + (i - 1) * PageSize) = 42; // map RSS memory
+ EXPECT_EQ(CountMapped(alignedPtr, i), 1ul);
+
+ ReleaseRss(alignedPtr, i);
+ EXPECT_EQ(CountMapped(alignedPtr, i), 0ul) << "number of pages " << i; // no RSS memory allocated
+
+ free(rawPtr);
+ }
+ }
+
+ TEST(StackUtilsTest, RssReleaseSeveralPages) {
+ char *rawPtr, *alignedPtr = nullptr;
+
+ for (uint64_t i : {1, 2, 5, 8}) {
+ EXPECT_TRUE(GetAlignedMemory(i, rawPtr, alignedPtr));
+ EXPECT_TRUE(rawPtr);
+ EXPECT_TRUE(alignedPtr);
+ EXPECT_FALSE((uint64_t)alignedPtr & PageSizeMask);
+
+ ReleaseRss(alignedPtr, i); // allocator can provide reused memory with RSS memory on it
+ EXPECT_EQ(CountMapped(alignedPtr, i), 0ul); // no RSS memory allocated
+
+ for (uint64_t page = 0; page < i; ++page) {
+ *(alignedPtr + page * PageSize) = 42; // map RSS memory
+ EXPECT_EQ(CountMapped(alignedPtr, page + 1), page + 1);
+ }
+
+ const uint64_t pagesToKeep = (i > 2) ? 2 : i;
+
+ ReleaseRss(alignedPtr, i - pagesToKeep);
+ EXPECT_EQ(CountMapped(alignedPtr, i), pagesToKeep) << "number of pages " << i; // no RSS memory allocated
+
+ free(rawPtr);
+ }
+ }
+
+#endif
+
+}
+
diff --git a/library/cpp/coroutine/engine/stack/ut/ya.make b/library/cpp/coroutine/engine/stack/ut/ya.make
index f2986487fd..65c5af9b7f 100644
--- a/library/cpp/coroutine/engine/stack/ut/ya.make
+++ b/library/cpp/coroutine/engine/stack/ut/ya.make
@@ -1,17 +1,17 @@
-GTEST()
-
-OWNER(g:balancer)
-
-SRCS(
- stack_allocator_ut.cpp
- stack_guards_ut.cpp
- stack_pool_ut.cpp
- stack_ut.cpp
- stack_utils_ut.cpp
-)
-
-PEERDIR(
- library/cpp/coroutine/engine
-)
-
-END() \ No newline at end of file
+GTEST()
+
+OWNER(g:balancer)
+
+SRCS(
+ stack_allocator_ut.cpp
+ stack_guards_ut.cpp
+ stack_pool_ut.cpp
+ stack_ut.cpp
+ stack_utils_ut.cpp
+)
+
+PEERDIR(
+ library/cpp/coroutine/engine
+)
+
+END() \ No newline at end of file
diff --git a/library/cpp/coroutine/engine/trampoline.cpp b/library/cpp/coroutine/engine/trampoline.cpp
index 2898ba562a..10ea69ddc3 100644
--- a/library/cpp/coroutine/engine/trampoline.cpp
+++ b/library/cpp/coroutine/engine/trampoline.cpp
@@ -1,8 +1,8 @@
#include "impl.h"
#include "trampoline.h"
-#include "stack/stack_allocator.h"
-
+#include "stack/stack_allocator.h"
+
#include <util/system/info.h>
#include <util/system/protect.h>
#include <util/system/valgrind.h>
@@ -11,11 +11,11 @@
#include <cstdlib>
#include <util/stream/format.h>
-
+
namespace NCoro {
TTrampoline::TTrampoline(NStack::IAllocator& allocator, ui32 stackSize, TFunc f, TCont* cont) noexcept
- : Stack_(allocator, stackSize, cont->Name())
+ : Stack_(allocator, stackSize, cont->Name())
, Clo_{this, Stack_.Get(), cont->Name()}
, Ctx_(Clo_)
, Func_(std::move(f))
diff --git a/library/cpp/coroutine/engine/trampoline.h b/library/cpp/coroutine/engine/trampoline.h
index af45d14c87..37b61cf015 100644
--- a/library/cpp/coroutine/engine/trampoline.h
+++ b/library/cpp/coroutine/engine/trampoline.h
@@ -1,8 +1,8 @@
#pragma once
-#include "stack/stack_common.h"
-#include "stack/stack.h"
-
+#include "stack/stack_common.h"
+#include "stack/stack.h"
+
#include <util/generic/noncopyable.h>
#include <util/generic/ptr.h>
#include <util/system/context.h>
@@ -17,8 +17,8 @@ typedef void (*TContFunc)(TCont*, void*);
namespace NCoro {
- namespace NStack {
- class IAllocator;
+ namespace NStack {
+ class IAllocator;
}
class TTrampoline : public ITrampoLine, TNonCopyable {
@@ -26,8 +26,8 @@ namespace NCoro {
typedef std::function<void (TCont*)> TFunc;
TTrampoline(
- NCoro::NStack::IAllocator& allocator,
- uint32_t stackSize,
+ NCoro::NStack::IAllocator& allocator,
+ uint32_t stackSize,
TFunc f,
TCont* cont
) noexcept;
@@ -51,7 +51,7 @@ namespace NCoro {
private:
const char* ContName() const noexcept;
private:
- NStack::TStackHolder Stack_;
+ NStack::TStackHolder Stack_;
const TContClosure Clo_;
TExceptionSafeContext Ctx_;
TFunc Func_;
diff --git a/library/cpp/coroutine/engine/ya.make b/library/cpp/coroutine/engine/ya.make
index fc31f25a8e..8c20b9afc3 100644
--- a/library/cpp/coroutine/engine/ya.make
+++ b/library/cpp/coroutine/engine/ya.make
@@ -6,7 +6,7 @@ OWNER(
)
GENERATE_ENUM_SERIALIZATION(poller.h)
-GENERATE_ENUM_SERIALIZATION(stack/stack_common.h)
+GENERATE_ENUM_SERIALIZATION(stack/stack_common.h)
PEERDIR(
contrib/libs/libc_compat
@@ -21,11 +21,11 @@ SRCS(
network.cpp
poller.cpp
sockpool.cpp
- stack/stack.cpp
- stack/stack_allocator.cpp
- stack/stack_guards.cpp
- stack/stack_storage.cpp
- stack/stack_utils.cpp
+ stack/stack.cpp
+ stack/stack_allocator.cpp
+ stack/stack_guards.cpp
+ stack/stack_storage.cpp
+ stack/stack_utils.cpp
trampoline.cpp
)