diff options
author | alexeylaptev <alexeylaptev@yandex-team.ru> | 2022-02-10 16:50:06 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:50:06 +0300 |
commit | 13dbd0acb78595551b843005d6bd021bdc1a859b (patch) | |
tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 /library/cpp/coroutine/engine/stack/stack_pool.inl | |
parent | 3106abc0443cda789ce4968aeee63d3a8fcc7d85 (diff) | |
download | ydb-13dbd0acb78595551b843005d6bd021bdc1a859b.tar.gz |
Restoring authorship annotation for <alexeylaptev@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'library/cpp/coroutine/engine/stack/stack_pool.inl')
-rw-r--r-- | library/cpp/coroutine/engine/stack/stack_pool.inl | 264 |
1 files changed, 132 insertions, 132 deletions
diff --git a/library/cpp/coroutine/engine/stack/stack_pool.inl b/library/cpp/coroutine/engine/stack/stack_pool.inl index 8a81514158..6e08e05a48 100644 --- a/library/cpp/coroutine/engine/stack/stack_pool.inl +++ b/library/cpp/coroutine/engine/stack/stack_pool.inl @@ -1,132 +1,132 @@ -#include "stack_storage.h" -#include "stack_utils.h" - - -namespace NCoro::NStack { - - template<typename TGuard> - TPool<TGuard>::TPool(uint64_t stackSize, const TPoolAllocatorSettings& settings, const TGuard& guard) - : StackSize_(stackSize) - , RssPagesToKeep_(IsSmallStack() ? settings.SmallStackRssPagesToKeep : settings.RssPagesToKeep) - , Guard_(guard) - , ChunkSize_(Guard_.GetPageAlignedSize() + StackSize_ * settings.StacksPerChunk) - { - Y_ASSERT(RssPagesToKeep_); - if (!RssPagesToKeep_) { - RssPagesToKeep_ = 1; // at least guard should be kept - } - - const uint64_t stackSizeInPages = stackSize / PageSize; - Y_ASSERT(stackSizeInPages >= RssPagesToKeep_); - if (stackSizeInPages < RssPagesToKeep_) { - RssPagesToKeep_ = stackSizeInPages; // keep all stack pages - } - - Y_ASSERT(StackSize_ && !(StackSize_ & PageSizeMask)); // stack size is not zero and page aligned - Y_ASSERT(Guard_.GetSize() < StackSize_); // stack has enough space to place guard - Y_ASSERT(stackSizeInPages >= RssPagesToKeep_); - - Storage_ = MakeHolder<TStorage>(StackSize_, RssPagesToKeep_, settings.ReleaseRate); - - AllocNewMemoryChunk(); - } - - template<typename TGuard> - TPool<TGuard>::TPool(TPool&& other) noexcept = default; - - template<typename TGuard> - TPool<TGuard>::~TPool() { - if (!Memory_.empty()) { - Y_ASSERT(NextToAlloc_ && StackSize_); - - for (const auto& chunk : Memory_) { - Y_ASSERT(chunk.Raw && chunk.Aligned); - - if (Guard_.ShouldRemoveProtectionBeforeFree()) { - Guard_.RemoveProtection(chunk.Aligned, Guard_.GetPageAlignedSize()); // first page in chunk - - const char* endOfStacksMemory = chunk.Aligned + ChunkSize_; - for (char* i = chunk.Aligned + Guard_.GetPageAlignedSize(); i < endOfStacksMemory; i += StackSize_) { - Guard_.RemoveProtection(i, StackSize_); - } - } - - free(chunk.Raw); - } - } - } - - template<typename TGuard> - NDetails::TStack TPool<TGuard>::AllocStack(const char* name) { - Y_ASSERT(!Memory_.empty()); - - if (!Storage_->IsEmpty()) { - return Storage_->GetStack(Guard_, name); - } else { - ++NumOfAllocated_; - return AllocNewStack(name); - } - } - - template<typename TGuard> - void TPool<TGuard>::FreeStack(NDetails::TStack& stack) { - Y_ASSERT(Storage_->Size() < ((ChunkSize_ - Guard_.GetPageAlignedSize()) / StackSize_) * Memory_.size()); - Y_ASSERT(IsStackFromThisPool(stack)); - - Storage_->ReturnStack(stack); - } - - template<typename TGuard> - uint64_t TPool<TGuard>::GetReleasedSize() const noexcept { - return Storage_->GetReleasedSize(); - } - template<typename TGuard> - uint64_t TPool<TGuard>::GetFullSize() const noexcept { - return Storage_->GetFullSize(); - } - - template<typename TGuard> - void TPool<TGuard>::AllocNewMemoryChunk() { - const uint64_t totalSizeInPages = ChunkSize_ / PageSize; - - TMemory memory; - const auto res = GetAlignedMemory(totalSizeInPages, memory.Raw, memory.Aligned); - Y_VERIFY(res, "Failed to allocate memory for coro stack pool"); - - NextToAlloc_ = memory.Aligned + Guard_.GetPageAlignedSize(); // skip first guard page - Guard_.Protect(memory.Aligned, Guard_.GetPageAlignedSize(), false); // protect first guard page - - Memory_.push_back(std::move(memory)); - } - - template<typename TGuard> - bool TPool<TGuard>::IsSmallStack() const noexcept { - return StackSize_ / PageSize <= SmallStackMaxSizeInPages; - } - - template<typename TGuard> - bool TPool<TGuard>::IsStackFromThisPool(const NDetails::TStack& stack) const noexcept { - for (const auto& chunk : Memory_) { - const char* endOfStacksMemory = chunk.Aligned + ChunkSize_; - if (chunk.Raw <= stack.GetRawMemory() && stack.GetRawMemory() < endOfStacksMemory) { - return true; - } - } - return false; - } - - template<typename TGuard> - NDetails::TStack TPool<TGuard>::AllocNewStack(const char* name) { - if (NextToAlloc_ + StackSize_ > Memory_.rbegin()->Aligned + ChunkSize_) { - AllocNewMemoryChunk(); // also sets NextToAlloc_ to first stack position in new allocated chunk of memory - } - Y_ASSERT(NextToAlloc_ + StackSize_ <= Memory_.rbegin()->Aligned + ChunkSize_); - - char* newStack = NextToAlloc_; - NextToAlloc_ += StackSize_; - - Guard_.Protect(newStack, StackSize_, true); - return NDetails::TStack{newStack, newStack, StackSize_, name}; - } - -} +#include "stack_storage.h" +#include "stack_utils.h" + + +namespace NCoro::NStack { + + template<typename TGuard> + TPool<TGuard>::TPool(uint64_t stackSize, const TPoolAllocatorSettings& settings, const TGuard& guard) + : StackSize_(stackSize) + , RssPagesToKeep_(IsSmallStack() ? settings.SmallStackRssPagesToKeep : settings.RssPagesToKeep) + , Guard_(guard) + , ChunkSize_(Guard_.GetPageAlignedSize() + StackSize_ * settings.StacksPerChunk) + { + Y_ASSERT(RssPagesToKeep_); + if (!RssPagesToKeep_) { + RssPagesToKeep_ = 1; // at least guard should be kept + } + + const uint64_t stackSizeInPages = stackSize / PageSize; + Y_ASSERT(stackSizeInPages >= RssPagesToKeep_); + if (stackSizeInPages < RssPagesToKeep_) { + RssPagesToKeep_ = stackSizeInPages; // keep all stack pages + } + + Y_ASSERT(StackSize_ && !(StackSize_ & PageSizeMask)); // stack size is not zero and page aligned + Y_ASSERT(Guard_.GetSize() < StackSize_); // stack has enough space to place guard + Y_ASSERT(stackSizeInPages >= RssPagesToKeep_); + + Storage_ = MakeHolder<TStorage>(StackSize_, RssPagesToKeep_, settings.ReleaseRate); + + AllocNewMemoryChunk(); + } + + template<typename TGuard> + TPool<TGuard>::TPool(TPool&& other) noexcept = default; + + template<typename TGuard> + TPool<TGuard>::~TPool() { + if (!Memory_.empty()) { + Y_ASSERT(NextToAlloc_ && StackSize_); + + for (const auto& chunk : Memory_) { + Y_ASSERT(chunk.Raw && chunk.Aligned); + + if (Guard_.ShouldRemoveProtectionBeforeFree()) { + Guard_.RemoveProtection(chunk.Aligned, Guard_.GetPageAlignedSize()); // first page in chunk + + const char* endOfStacksMemory = chunk.Aligned + ChunkSize_; + for (char* i = chunk.Aligned + Guard_.GetPageAlignedSize(); i < endOfStacksMemory; i += StackSize_) { + Guard_.RemoveProtection(i, StackSize_); + } + } + + free(chunk.Raw); + } + } + } + + template<typename TGuard> + NDetails::TStack TPool<TGuard>::AllocStack(const char* name) { + Y_ASSERT(!Memory_.empty()); + + if (!Storage_->IsEmpty()) { + return Storage_->GetStack(Guard_, name); + } else { + ++NumOfAllocated_; + return AllocNewStack(name); + } + } + + template<typename TGuard> + void TPool<TGuard>::FreeStack(NDetails::TStack& stack) { + Y_ASSERT(Storage_->Size() < ((ChunkSize_ - Guard_.GetPageAlignedSize()) / StackSize_) * Memory_.size()); + Y_ASSERT(IsStackFromThisPool(stack)); + + Storage_->ReturnStack(stack); + } + + template<typename TGuard> + uint64_t TPool<TGuard>::GetReleasedSize() const noexcept { + return Storage_->GetReleasedSize(); + } + template<typename TGuard> + uint64_t TPool<TGuard>::GetFullSize() const noexcept { + return Storage_->GetFullSize(); + } + + template<typename TGuard> + void TPool<TGuard>::AllocNewMemoryChunk() { + const uint64_t totalSizeInPages = ChunkSize_ / PageSize; + + TMemory memory; + const auto res = GetAlignedMemory(totalSizeInPages, memory.Raw, memory.Aligned); + Y_VERIFY(res, "Failed to allocate memory for coro stack pool"); + + NextToAlloc_ = memory.Aligned + Guard_.GetPageAlignedSize(); // skip first guard page + Guard_.Protect(memory.Aligned, Guard_.GetPageAlignedSize(), false); // protect first guard page + + Memory_.push_back(std::move(memory)); + } + + template<typename TGuard> + bool TPool<TGuard>::IsSmallStack() const noexcept { + return StackSize_ / PageSize <= SmallStackMaxSizeInPages; + } + + template<typename TGuard> + bool TPool<TGuard>::IsStackFromThisPool(const NDetails::TStack& stack) const noexcept { + for (const auto& chunk : Memory_) { + const char* endOfStacksMemory = chunk.Aligned + ChunkSize_; + if (chunk.Raw <= stack.GetRawMemory() && stack.GetRawMemory() < endOfStacksMemory) { + return true; + } + } + return false; + } + + template<typename TGuard> + NDetails::TStack TPool<TGuard>::AllocNewStack(const char* name) { + if (NextToAlloc_ + StackSize_ > Memory_.rbegin()->Aligned + ChunkSize_) { + AllocNewMemoryChunk(); // also sets NextToAlloc_ to first stack position in new allocated chunk of memory + } + Y_ASSERT(NextToAlloc_ + StackSize_ <= Memory_.rbegin()->Aligned + ChunkSize_); + + char* newStack = NextToAlloc_; + NextToAlloc_ += StackSize_; + + Guard_.Protect(newStack, StackSize_, true); + return NDetails::TStack{newStack, newStack, StackSize_, name}; + } + +} |