diff options
author | single <single@yandex-team.ru> | 2022-02-10 16:50:30 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:50:30 +0300 |
commit | f7835298a8840c8e5d98715bf23efa9c7e03b9c4 (patch) | |
tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 /library/cpp/lfalloc | |
parent | 8ae96df130bbede609c3504aa9af1bc6ff5361b3 (diff) | |
download | ydb-f7835298a8840c8e5d98715bf23efa9c7e03b9c4.tar.gz |
Restoring authorship annotation for <single@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'library/cpp/lfalloc')
-rw-r--r-- | library/cpp/lfalloc/lf_allocX64.h | 120 |
1 files changed, 60 insertions, 60 deletions
diff --git a/library/cpp/lfalloc/lf_allocX64.h b/library/cpp/lfalloc/lf_allocX64.h index 588b051815..fd2a906d6f 100644 --- a/library/cpp/lfalloc/lf_allocX64.h +++ b/library/cpp/lfalloc/lf_allocX64.h @@ -390,53 +390,53 @@ static char* AllocWithMMap(uintptr_t sz, EMMapMode mode) { return largeBlock; } -enum class ELarge : ui8 { - Free = 0, // block in free cache - Alloc = 1, // block is allocated - Gone = 2, // block was unmapped -}; - -struct TLargeBlk { - - static TLargeBlk* As(void *raw) { - return reinterpret_cast<TLargeBlk*>((char*)raw - 4096ll); - } - - static const TLargeBlk* As(const void *raw) { - return reinterpret_cast<const TLargeBlk*>((const char*)raw - 4096ll); - } - - void SetSize(size_t bytes, size_t pages) { - Pages = pages; - Bytes = bytes; - } - - void Mark(ELarge state) { - const ui64 marks[] = { - 0x8b38aa5ca4953c98, // ELarge::Free - 0xf916d33584eb5087, // ELarge::Alloc - 0xd33b0eca7651bc3f // ELarge::Gone - }; - - Token = size_t(marks[ui8(state)]); - } - - size_t Pages; // Total pages allocated with mmap like call - size_t Bytes; // Actually requested bytes by user - size_t Token; // Block state token, see ELarge enum. -}; - - -static void LargeBlockUnmap(void* p, size_t pages) { - const auto bytes = (pages + 1) * uintptr_t(4096); - - IncrementCounter(CT_MUNMAP, bytes); +enum class ELarge : ui8 { + Free = 0, // block in free cache + Alloc = 1, // block is allocated + Gone = 2, // block was unmapped +}; + +struct TLargeBlk { + + static TLargeBlk* As(void *raw) { + return reinterpret_cast<TLargeBlk*>((char*)raw - 4096ll); + } + + static const TLargeBlk* As(const void *raw) { + return reinterpret_cast<const TLargeBlk*>((const char*)raw - 4096ll); + } + + void SetSize(size_t bytes, size_t pages) { + Pages = pages; + Bytes = bytes; + } + + void Mark(ELarge state) { + const ui64 marks[] = { + 0x8b38aa5ca4953c98, // ELarge::Free + 0xf916d33584eb5087, // ELarge::Alloc + 0xd33b0eca7651bc3f // ELarge::Gone + }; + + Token = size_t(marks[ui8(state)]); + } + + size_t Pages; // Total pages allocated with mmap like call + size_t Bytes; // Actually requested bytes by user + size_t Token; // Block state token, see ELarge enum. +}; + + +static void LargeBlockUnmap(void* p, size_t pages) { + const auto bytes = (pages + 1) * uintptr_t(4096); + + IncrementCounter(CT_MUNMAP, bytes); IncrementCounter(CT_MUNMAP_CNT, 1); #ifdef _MSC_VER Y_ASSERT_NOBT(0); #else - TLargeBlk::As(p)->Mark(ELarge::Gone); - munmap((char*)p - 4096ll, bytes); + TLargeBlk::As(p)->Mark(ELarge::Gone); + munmap((char*)p - 4096ll, bytes); #endif } @@ -447,7 +447,7 @@ static int LB_LIMIT_TOTAL_SIZE = 500 * 1024 * 1024 / 4096; // do not keep more t static void* volatile lbFreePtrs[LB_BUF_HASH][LB_BUF_SIZE]; static TAtomic lbFreePageCount; - + static void* LargeBlockAlloc(size_t _nSize, ELFAllocCounter counter) { size_t pgCount = (_nSize + 4095) / 4096; #ifdef _MSC_VER @@ -466,16 +466,16 @@ static void* LargeBlockAlloc(size_t _nSize, ELFAllocCounter counter) { if (p == nullptr) continue; if (DoCas(&lbFreePtrs[lbHash][i], (void*)nullptr, p) == p) { - size_t realPageCount = TLargeBlk::As(p)->Pages; + size_t realPageCount = TLargeBlk::As(p)->Pages; if (realPageCount == pgCount) { AtomicAdd(lbFreePageCount, -pgCount); - TLargeBlk::As(p)->Mark(ELarge::Alloc); + TLargeBlk::As(p)->Mark(ELarge::Alloc); return p; } else { if (DoCas(&lbFreePtrs[lbHash][i], p, (void*)nullptr) != (void*)nullptr) { // block was freed while we were busy AtomicAdd(lbFreePageCount, -realPageCount); - LargeBlockUnmap(p, realPageCount); + LargeBlockUnmap(p, realPageCount); --i; } } @@ -484,8 +484,8 @@ static void* LargeBlockAlloc(size_t _nSize, ELFAllocCounter counter) { char* pRes = AllocWithMMap((pgCount + 1) * 4096ll, MM_HUGE); #endif pRes += 4096ll; - TLargeBlk::As(pRes)->SetSize(_nSize, pgCount); - TLargeBlk::As(pRes)->Mark(ELarge::Alloc); + TLargeBlk::As(pRes)->SetSize(_nSize, pgCount); + TLargeBlk::As(pRes)->Mark(ELarge::Alloc); return pRes; } @@ -498,9 +498,9 @@ static void FreeAllLargeBlockMem() { if (p == nullptr) continue; if (DoCas(&lbFreePtr[i], (void*)nullptr, p) == p) { - int pgCount = TLargeBlk::As(p)->Pages; + int pgCount = TLargeBlk::As(p)->Pages; AtomicAdd(lbFreePageCount, -pgCount); - LargeBlockUnmap(p, pgCount); + LargeBlockUnmap(p, pgCount); } } } @@ -513,9 +513,9 @@ static void LargeBlockFree(void* p, ELFAllocCounter counter) { #ifdef _MSC_VER VirtualFree((char*)p - 4096ll, 0, MEM_RELEASE); #else - size_t pgCount = TLargeBlk::As(p)->Pages; + size_t pgCount = TLargeBlk::As(p)->Pages; - TLargeBlk::As(p)->Mark(ELarge::Free); + TLargeBlk::As(p)->Mark(ELarge::Free); IncrementCounter(counter, pgCount * 4096ll); IncrementCounter(CT_SYSTEM_FREE, 4096ll); @@ -531,7 +531,7 @@ static void LargeBlockFree(void* p, ELFAllocCounter counter) { } } - LargeBlockUnmap(p, pgCount); + LargeBlockUnmap(p, pgCount); #endif } @@ -1644,23 +1644,23 @@ static Y_FORCE_INLINE void LFFree(void* p) { } } -static size_t LFGetSize(const void* p) { +static size_t LFGetSize(const void* p) { #if defined(LFALLOC_DBG) if (p == nullptr) return 0; - return GetAllocHeader(const_cast<void*>(p))->Size; + return GetAllocHeader(const_cast<void*>(p))->Size; #endif - uintptr_t chkOffset = ((const char*)p - ALLOC_START); + uintptr_t chkOffset = ((const char*)p - ALLOC_START); if (chkOffset >= N_MAX_WORKSET_SIZE) { if (p == nullptr) return 0; - return TLargeBlk::As(p)->Pages * 4096ll; + return TLargeBlk::As(p)->Pages * 4096ll; } - uintptr_t chunk = ((const char*)p - ALLOC_START) / N_CHUNK_SIZE; + uintptr_t chunk = ((const char*)p - ALLOC_START) / N_CHUNK_SIZE; ptrdiff_t nSizeIdx = chunkSizeIdx[chunk]; if (nSizeIdx <= 0) - return TLargeBlk::As(p)->Pages * 4096ll; + return TLargeBlk::As(p)->Pages * 4096ll; return nSizeIdxToSize[nSizeIdx]; } |