diff options
author | ironpeter <ironpeter@yandex-team.ru> | 2022-02-10 16:49:51 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:49:51 +0300 |
commit | ff97837ecc5972a00cb395483d8856566738375c (patch) | |
tree | b2e9b0b27c06242cc2390f3fe726bd2d40758c8f /util/thread | |
parent | ec31dbabb2178819f10e1dec8f2ae9b2ba551ab1 (diff) | |
download | ydb-ff97837ecc5972a00cb395483d8856566738375c.tar.gz |
Restoring authorship annotation for <ironpeter@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'util/thread')
-rw-r--r-- | util/thread/lfqueue.h | 218 | ||||
-rw-r--r-- | util/thread/lfstack.h | 92 |
2 files changed, 155 insertions, 155 deletions
diff --git a/util/thread/lfqueue.h b/util/thread/lfqueue.h index ab523631e4..510c91e788 100644 --- a/util/thread/lfqueue.h +++ b/util/thread/lfqueue.h @@ -1,12 +1,12 @@ #pragma once - + #include "fwd.h" #include <util/generic/ptr.h> -#include <util/system/atomic.h> +#include <util/system/atomic.h> #include <util/system/yassert.h> #include "lfstack.h" - + struct TDefaultLFCounter { template <class T> void IncCount(const T& data) { @@ -40,137 +40,137 @@ class TLockFreeQueue: public TNonCopyable { } TListNode* volatile Next; - T Data; - }; - + T Data; + }; + // using inheritance to be able to use 0 bytes for TCounter when we don't need one struct TRootNode: public TCounter { TListNode* volatile PushQueue; TListNode* volatile PopQueue; TListNode* volatile ToDelete; TRootNode* volatile NextFree; - + TRootNode() : PushQueue(nullptr) , PopQueue(nullptr) , ToDelete(nullptr) , NextFree(nullptr) - { - } + { + } void CopyCounter(TRootNode* x) { *(TCounter*)this = *(TCounter*)x; } - }; - + }; + static void EraseList(TListNode* n) { - while (n) { + while (n) { TListNode* keepNext = AtomicGet(n->Next); - delete n; - n = keepNext; - } - } + delete n; + n = keepNext; + } + } alignas(64) TRootNode* volatile JobQueue; alignas(64) volatile TAtomic FreememCounter; alignas(64) volatile TAtomic FreeingTaskCounter; alignas(64) TRootNode* volatile FreePtr; - + void TryToFreeAsyncMemory() { TAtomic keepCounter = AtomicAdd(FreeingTaskCounter, 0); TRootNode* current = AtomicGet(FreePtr); if (current == nullptr) - return; + return; if (AtomicAdd(FreememCounter, 0) == 1) { - // we are the last thread, try to cleanup + // we are the last thread, try to cleanup // check if another thread have cleaned up if (keepCounter != AtomicAdd(FreeingTaskCounter, 0)) { return; } if (AtomicCas(&FreePtr, (TRootNode*)nullptr, current)) { - // free list - while (current) { + // free list + while (current) { TRootNode* p = AtomicGet(current->NextFree); EraseList(AtomicGet(current->ToDelete)); - delete current; - current = p; - } + delete current; + current = p; + } AtomicAdd(FreeingTaskCounter, 1); - } - } - } + } + } + } void AsyncRef() { AtomicAdd(FreememCounter, 1); - } + } void AsyncUnref() { - TryToFreeAsyncMemory(); + TryToFreeAsyncMemory(); AtomicAdd(FreememCounter, -1); - } + } void AsyncDel(TRootNode* toDelete, TListNode* lst) { AtomicSet(toDelete->ToDelete, lst); for (;;) { AtomicSet(toDelete->NextFree, AtomicGet(FreePtr)); if (AtomicCas(&FreePtr, toDelete, AtomicGet(toDelete->NextFree))) - break; - } - } + break; + } + } void AsyncUnref(TRootNode* toDelete, TListNode* lst) { - TryToFreeAsyncMemory(); + TryToFreeAsyncMemory(); if (AtomicAdd(FreememCounter, -1) == 0) { - // no other operations in progress, can safely reclaim memory - EraseList(lst); - delete toDelete; - } else { - // Dequeue()s in progress, put node to free list - AsyncDel(toDelete, lst); - } - } - + // no other operations in progress, can safely reclaim memory + EraseList(lst); + delete toDelete; + } else { + // Dequeue()s in progress, put node to free list + AsyncDel(toDelete, lst); + } + } + struct TListInvertor { TListNode* Copy; TListNode* Tail; TListNode* PrevFirst; - + TListInvertor() : Copy(nullptr) , Tail(nullptr) , PrevFirst(nullptr) - { - } + { + } ~TListInvertor() { - EraseList(Copy); - } + EraseList(Copy); + } void CopyWasUsed() { Copy = nullptr; Tail = nullptr; PrevFirst = nullptr; - } + } void DoCopy(TListNode* ptr) { TListNode* newFirst = ptr; TListNode* newCopy = nullptr; TListNode* newTail = nullptr; - while (ptr) { - if (ptr == PrevFirst) { - // short cut, we have copied this part already + while (ptr) { + if (ptr == PrevFirst) { + // short cut, we have copied this part already AtomicSet(Tail->Next, newCopy); - newCopy = Copy; + newCopy = Copy; Copy = nullptr; // do not destroy prev try - if (!newTail) - newTail = Tail; // tried to invert same list - break; - } + if (!newTail) + newTail = Tail; // tried to invert same list + break; + } TListNode* newElem = new TListNode(ptr->Data, newCopy); - newCopy = newElem; + newCopy = newElem; ptr = AtomicGet(ptr->Next); - if (!newTail) - newTail = newElem; - } - EraseList(Copy); // copy was useless - Copy = newCopy; - PrevFirst = newFirst; - Tail = newTail; - } - }; - + if (!newTail) + newTail = newElem; + } + EraseList(Copy); // copy was useless + Copy = newCopy; + PrevFirst = newFirst; + Tail = newTail; + } + }; + void EnqueueImpl(TListNode* head, TListNode* tail) { TRootNode* newRoot = new TRootNode; AsyncRef(); @@ -224,21 +224,21 @@ class TLockFreeQueue: public TNonCopyable { return lst; } -public: +public: TLockFreeQueue() - : JobQueue(new TRootNode) - , FreememCounter(0) + : JobQueue(new TRootNode) + , FreememCounter(0) , FreeingTaskCounter(0) , FreePtr(nullptr) - { - } + { + } ~TLockFreeQueue() { AsyncRef(); AsyncUnref(); // should free FreeList - EraseList(JobQueue->PushQueue); - EraseList(JobQueue->PopQueue); - delete JobQueue; - } + EraseList(JobQueue->PushQueue); + EraseList(JobQueue->PopQueue); + delete JobQueue; + } template <typename U> void Enqueue(U&& data) { TListNode* newNode = new TListNode(std::forward<U>(data)); @@ -268,21 +268,21 @@ public: for (++i; i != dataEnd; ++i) { TListNode* nextNode = node; node = new TListNode(*i, nextNode); - } + } EnqueueImpl(node, tail); - } + } bool Dequeue(T* data) { TRootNode* newRoot = nullptr; - TListInvertor listInvertor; - AsyncRef(); - for (;;) { + TListInvertor listInvertor; + AsyncRef(); + for (;;) { TRootNode* curRoot = AtomicGet(JobQueue); TListNode* tail = AtomicGet(curRoot->PopQueue); - if (tail) { - // has elems to pop - if (!newRoot) - newRoot = new TRootNode; - + if (tail) { + // has elems to pop + if (!newRoot) + newRoot = new TRootNode; + AtomicSet(newRoot->PushQueue, AtomicGet(curRoot->PushQueue)); AtomicSet(newRoot->PopQueue, AtomicGet(tail->Next)); newRoot->CopyCounter(curRoot); @@ -291,19 +291,19 @@ public: if (AtomicCas(&JobQueue, newRoot, curRoot)) { *data = std::move(tail->Data); AtomicSet(tail->Next, nullptr); - AsyncUnref(curRoot, tail); - return true; - } - continue; - } + AsyncUnref(curRoot, tail); + return true; + } + continue; + } if (AtomicGet(curRoot->PushQueue) == nullptr) { - delete newRoot; - AsyncUnref(); - return false; // no elems to pop - } - - if (!newRoot) - newRoot = new TRootNode; + delete newRoot; + AsyncUnref(); + return false; // no elems to pop + } + + if (!newRoot) + newRoot = new TRootNode; AtomicSet(newRoot->PushQueue, nullptr); listInvertor.DoCopy(AtomicGet(curRoot->PushQueue)); AtomicSet(newRoot->PopQueue, listInvertor.Copy); @@ -311,13 +311,13 @@ public: Y_ASSERT(AtomicGet(curRoot->PopQueue) == nullptr); if (AtomicCas(&JobQueue, newRoot, curRoot)) { newRoot = nullptr; - listInvertor.CopyWasUsed(); + listInvertor.CopyWasUsed(); AsyncDel(curRoot, AtomicGet(curRoot->PushQueue)); - } else { + } else { AtomicSet(newRoot->PopQueue, nullptr); - } - } - } + } + } + } template <typename TCollection> void DequeueAll(TCollection* res) { AsyncRef(); @@ -344,12 +344,12 @@ public: AsyncUnref(curRoot, toDeleteHead); } bool IsEmpty() { - AsyncRef(); + AsyncRef(); TRootNode* curRoot = AtomicGet(JobQueue); bool res = AtomicGet(curRoot->PushQueue) == nullptr && AtomicGet(curRoot->PopQueue) == nullptr; - AsyncUnref(); - return res; - } + AsyncUnref(); + return res; + } TCounter GetCounter() { AsyncRef(); TRootNode* curRoot = AtomicGet(JobQueue); @@ -357,7 +357,7 @@ public: AsyncUnref(); return res; } -}; +}; template <class T, class TCounter> class TAutoLockFreeQueue { diff --git a/util/thread/lfstack.h b/util/thread/lfstack.h index ca3d95f3c3..a85b6100b6 100644 --- a/util/thread/lfstack.h +++ b/util/thread/lfstack.h @@ -1,16 +1,16 @@ #pragma once - + #include <util/generic/noncopyable.h> -#include <util/system/atomic.h> - -////////////////////////////// -// lock free lifo stack +#include <util/system/atomic.h> + +////////////////////////////// +// lock free lifo stack template <class T> class TLockFreeStack: TNonCopyable { struct TNode { - T Value; + T Value; TNode* Next; - + TNode() = default; template <class U> @@ -19,29 +19,29 @@ class TLockFreeStack: TNonCopyable { , Next(nullptr) { } - }; - + }; + TNode* Head; TNode* FreePtr; - TAtomic DequeueCount; - + TAtomic DequeueCount; + void TryToFreeMemory() { TNode* current = AtomicGet(FreePtr); - if (!current) - return; + if (!current) + return; if (AtomicAdd(DequeueCount, 0) == 1) { - // node current is in free list, we are the last thread so try to cleanup + // node current is in free list, we are the last thread so try to cleanup if (AtomicCas(&FreePtr, (TNode*)nullptr, current)) - EraseList(current); - } - } + EraseList(current); + } + } void EraseList(TNode* volatile p) { - while (p) { + while (p) { TNode* next = p->Next; - delete p; - p = next; - } - } + delete p; + p = next; + } + } void EnqueueImpl(TNode* volatile head, TNode* volatile tail) { for (;;) { tail->Next = AtomicGet(Head); @@ -55,7 +55,7 @@ class TLockFreeStack: TNonCopyable { EnqueueImpl(node, node); } -public: +public: TLockFreeStack() : Head(nullptr) , FreePtr(nullptr) @@ -63,9 +63,9 @@ public: { } ~TLockFreeStack() { - EraseList(Head); - EraseList(FreePtr); - } + EraseList(Head); + EraseList(FreePtr); + } void Enqueue(const T& t) { EnqueueImpl(t); @@ -83,7 +83,7 @@ public: void EnqueueAll(TIter dataBegin, TIter dataEnd) { if (dataBegin == dataEnd) { return; - } + } TIter i = dataBegin; TNode* volatile node = new TNode(*i); TNode* volatile tail = node; @@ -94,33 +94,33 @@ public: node->Next = nextNode; } EnqueueImpl(node, tail); - } + } bool Dequeue(T* res) { AtomicAdd(DequeueCount, 1); for (TNode* current = AtomicGet(Head); current; current = AtomicGet(Head)) { if (AtomicCas(&Head, AtomicGet(current->Next), current)) { *res = std::move(current->Value); - // delete current; // ABA problem - // even more complex node deletion - TryToFreeMemory(); + // delete current; // ABA problem + // even more complex node deletion + TryToFreeMemory(); if (AtomicAdd(DequeueCount, -1) == 0) { - // no other Dequeue()s, can safely reclaim memory - delete current; - } else { - // Dequeue()s in progress, put node to free list + // no other Dequeue()s, can safely reclaim memory + delete current; + } else { + // Dequeue()s in progress, put node to free list for (;;) { AtomicSet(current->Next, AtomicGet(FreePtr)); if (AtomicCas(&FreePtr, current, current->Next)) - break; - } - } - return true; - } - } - TryToFreeMemory(); + break; + } + } + return true; + } + } + TryToFreeMemory(); AtomicAdd(DequeueCount, -1); - return false; - } + return false; + } // add all elements to *res // elements are returned in order of dequeue (top to bottom; see example in unittest) template <typename TCollection> @@ -184,5 +184,5 @@ public: bool IsEmpty() { AtomicAdd(DequeueCount, 0); // mem barrier return AtomicGet(Head) == nullptr; // without lock, so result is approximate - } -}; + } +}; |