diff options
author | snaury <snaury@ydb.tech> | 2023-02-09 11:23:12 +0300 |
---|---|---|
committer | snaury <snaury@ydb.tech> | 2023-02-09 11:23:12 +0300 |
commit | 407b9e52eea66d83508fb8abd1e44ae52afcd9b6 (patch) | |
tree | c1540d69056ce4be1f153f649fbba484acea994c | |
parent | bbb79f5a581e9a3e2111f2823bcf16288e60e98c (diff) | |
download | ydb-407b9e52eea66d83508fb8abd1e44ae52afcd9b6.tar.gz |
Fix memory pool stats recalculation on rollback
-rw-r--r-- | ydb/core/tablet_flat/CMakeLists.darwin.txt | 1 | ||||
-rw-r--r-- | ydb/core/tablet_flat/CMakeLists.linux-aarch64.txt | 1 | ||||
-rw-r--r-- | ydb/core/tablet_flat/CMakeLists.linux.txt | 1 | ||||
-rw-r--r-- | ydb/core/tablet_flat/ut/CMakeLists.darwin.txt | 1 | ||||
-rw-r--r-- | ydb/core/tablet_flat/ut/CMakeLists.linux-aarch64.txt | 1 | ||||
-rw-r--r-- | ydb/core/tablet_flat/ut/CMakeLists.linux.txt | 1 | ||||
-rw-r--r-- | ydb/core/tablet_flat/ut_util/CMakeLists.darwin.txt | 72 | ||||
-rw-r--r-- | ydb/core/tablet_flat/ut_util/CMakeLists.linux-aarch64.txt | 75 | ||||
-rw-r--r-- | ydb/core/tablet_flat/ut_util/CMakeLists.linux.txt | 77 | ||||
-rw-r--r-- | ydb/core/tablet_flat/ut_util/CMakeLists.txt | 15 | ||||
-rw-r--r-- | ydb/core/tablet_flat/util_pool.h | 18 | ||||
-rw-r--r-- | ydb/core/tablet_flat/util_pool_ut.cpp | 19 |
12 files changed, 276 insertions, 6 deletions
diff --git a/ydb/core/tablet_flat/CMakeLists.darwin.txt b/ydb/core/tablet_flat/CMakeLists.darwin.txt index 7398f2a9698..56e4ae1c32d 100644 --- a/ydb/core/tablet_flat/CMakeLists.darwin.txt +++ b/ydb/core/tablet_flat/CMakeLists.darwin.txt @@ -11,6 +11,7 @@ add_subdirectory(test) add_subdirectory(ut) add_subdirectory(ut_large) add_subdirectory(ut_pg) +add_subdirectory(ut_util) add_library(ydb-core-tablet_flat) target_compile_options(ydb-core-tablet_flat PRIVATE diff --git a/ydb/core/tablet_flat/CMakeLists.linux-aarch64.txt b/ydb/core/tablet_flat/CMakeLists.linux-aarch64.txt index 03ca64a9895..168a2df75a6 100644 --- a/ydb/core/tablet_flat/CMakeLists.linux-aarch64.txt +++ b/ydb/core/tablet_flat/CMakeLists.linux-aarch64.txt @@ -11,6 +11,7 @@ add_subdirectory(test) add_subdirectory(ut) add_subdirectory(ut_large) add_subdirectory(ut_pg) +add_subdirectory(ut_util) add_library(ydb-core-tablet_flat) target_compile_options(ydb-core-tablet_flat PRIVATE diff --git a/ydb/core/tablet_flat/CMakeLists.linux.txt b/ydb/core/tablet_flat/CMakeLists.linux.txt index 03ca64a9895..168a2df75a6 100644 --- a/ydb/core/tablet_flat/CMakeLists.linux.txt +++ b/ydb/core/tablet_flat/CMakeLists.linux.txt @@ -11,6 +11,7 @@ add_subdirectory(test) add_subdirectory(ut) add_subdirectory(ut_large) add_subdirectory(ut_pg) +add_subdirectory(ut_util) add_library(ydb-core-tablet_flat) target_compile_options(ydb-core-tablet_flat PRIVATE diff --git a/ydb/core/tablet_flat/ut/CMakeLists.darwin.txt b/ydb/core/tablet_flat/ut/CMakeLists.darwin.txt index cc9d15d8d84..c6b2d3446f4 100644 --- a/ydb/core/tablet_flat/ut/CMakeLists.darwin.txt +++ b/ydb/core/tablet_flat/ut/CMakeLists.darwin.txt @@ -48,7 +48,6 @@ target_sources(ydb-core-tablet_flat-ut PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_table_part_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/flat_test_db.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/shared_handle_ut.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/util_pool_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/ut_self.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/ut_iterator.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/ut_memtable.cpp diff --git a/ydb/core/tablet_flat/ut/CMakeLists.linux-aarch64.txt b/ydb/core/tablet_flat/ut/CMakeLists.linux-aarch64.txt index 175c70b919c..2cbc8909529 100644 --- a/ydb/core/tablet_flat/ut/CMakeLists.linux-aarch64.txt +++ b/ydb/core/tablet_flat/ut/CMakeLists.linux-aarch64.txt @@ -51,7 +51,6 @@ target_sources(ydb-core-tablet_flat-ut PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_table_part_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/flat_test_db.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/shared_handle_ut.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/util_pool_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/ut_self.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/ut_iterator.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/ut_memtable.cpp diff --git a/ydb/core/tablet_flat/ut/CMakeLists.linux.txt b/ydb/core/tablet_flat/ut/CMakeLists.linux.txt index cd630a668b7..e0a1415129e 100644 --- a/ydb/core/tablet_flat/ut/CMakeLists.linux.txt +++ b/ydb/core/tablet_flat/ut/CMakeLists.linux.txt @@ -53,7 +53,6 @@ target_sources(ydb-core-tablet_flat-ut PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_table_part_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/flat_test_db.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/shared_handle_ut.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/util_pool_ut.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/ut_self.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/ut_iterator.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/ut_memtable.cpp diff --git a/ydb/core/tablet_flat/ut_util/CMakeLists.darwin.txt b/ydb/core/tablet_flat/ut_util/CMakeLists.darwin.txt new file mode 100644 index 00000000000..375c1fa7820 --- /dev/null +++ b/ydb/core/tablet_flat/ut_util/CMakeLists.darwin.txt @@ -0,0 +1,72 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-core-tablet_flat-ut_util) +target_include_directories(ydb-core-tablet_flat-ut_util PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat +) +target_link_libraries(ydb-core-tablet_flat-ut_util PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + ydb-core-tablet_flat +) +target_link_options(ydb-core-tablet_flat-ut_util PRIVATE + -Wl,-no_deduplicate + -Wl,-sdk_version,10.15 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-core-tablet_flat-ut_util PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/util_pool_ut.cpp +) +set_property( + TARGET + ydb-core-tablet_flat-ut_util + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-core-tablet_flat-ut_util + TEST_TARGET + ydb-core-tablet_flat-ut_util + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-core-tablet_flat-ut_util + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-core-tablet_flat-ut_util + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-core-tablet_flat-ut_util + PROPERTY + TIMEOUT + 600 +) +vcs_info(ydb-core-tablet_flat-ut_util) diff --git a/ydb/core/tablet_flat/ut_util/CMakeLists.linux-aarch64.txt b/ydb/core/tablet_flat/ut_util/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..c2fba7805f6 --- /dev/null +++ b/ydb/core/tablet_flat/ut_util/CMakeLists.linux-aarch64.txt @@ -0,0 +1,75 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-core-tablet_flat-ut_util) +target_include_directories(ydb-core-tablet_flat-ut_util PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat +) +target_link_libraries(ydb-core-tablet_flat-ut_util PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-lfalloc + cpp-testing-unittest_main + ydb-core-tablet_flat +) +target_link_options(ydb-core-tablet_flat-ut_util PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-core-tablet_flat-ut_util PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/util_pool_ut.cpp +) +set_property( + TARGET + ydb-core-tablet_flat-ut_util + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-core-tablet_flat-ut_util + TEST_TARGET + ydb-core-tablet_flat-ut_util + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-core-tablet_flat-ut_util + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-core-tablet_flat-ut_util + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-core-tablet_flat-ut_util + PROPERTY + TIMEOUT + 600 +) +vcs_info(ydb-core-tablet_flat-ut_util) diff --git a/ydb/core/tablet_flat/ut_util/CMakeLists.linux.txt b/ydb/core/tablet_flat/ut_util/CMakeLists.linux.txt new file mode 100644 index 00000000000..473fb6d1337 --- /dev/null +++ b/ydb/core/tablet_flat/ut_util/CMakeLists.linux.txt @@ -0,0 +1,77 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-core-tablet_flat-ut_util) +target_include_directories(ydb-core-tablet_flat-ut_util PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat +) +target_link_libraries(ydb-core-tablet_flat-ut_util PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache + library-cpp-cpuid_check + cpp-testing-unittest_main + ydb-core-tablet_flat +) +target_link_options(ydb-core-tablet_flat-ut_util PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-core-tablet_flat-ut_util PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/util_pool_ut.cpp +) +set_property( + TARGET + ydb-core-tablet_flat-ut_util + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-core-tablet_flat-ut_util + TEST_TARGET + ydb-core-tablet_flat-ut_util + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-core-tablet_flat-ut_util + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-core-tablet_flat-ut_util + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-core-tablet_flat-ut_util + PROPERTY + TIMEOUT + 600 +) +vcs_info(ydb-core-tablet_flat-ut_util) diff --git a/ydb/core/tablet_flat/ut_util/CMakeLists.txt b/ydb/core/tablet_flat/ut_util/CMakeLists.txt new file mode 100644 index 00000000000..5bb4faffb40 --- /dev/null +++ b/ydb/core/tablet_flat/ut_util/CMakeLists.txt @@ -0,0 +1,15 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND UNIX AND NOT APPLE AND NOT ANDROID) + include(CMakeLists.linux-aarch64.txt) +elseif (APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin.txt) +elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND UNIX AND NOT APPLE AND NOT ANDROID) + include(CMakeLists.linux.txt) +endif() diff --git a/ydb/core/tablet_flat/util_pool.h b/ydb/core/tablet_flat/util_pool.h index e3cb17a5ee1..db6dbbc4103 100644 --- a/ydb/core/tablet_flat/util_pool.h +++ b/ydb/core/tablet_flat/util_pool.h @@ -111,7 +111,9 @@ namespace NKikimr::NUtil { TMemoryPool(size_t initial) : First(AllocateChunk(Max(initial, TChunk::ChunkHeaderSize() + PLATFORM_DATA_ALIGN))) , Current(First) - { } + { + Total_ = Current->ChunkSize(); + } ~TMemoryPool() { TChunk* chunk = First; @@ -157,6 +159,10 @@ namespace NKikimr::NUtil { RollbackState_.reset(); } + size_t Total() const noexcept { + return Total_; + } + size_t Used() const noexcept { return Used_ + Current->Used(); } @@ -190,6 +196,7 @@ namespace NKikimr::NUtil { Y_VERIFY(!Current->Next); size_t hint = Max(AlignUp<size_t>(sizeof(TChunk), PLATFORM_DATA_ALIGN) + size, Current->ChunkSize() + 1); TChunk* next = AllocateChunk(hint); + Total_ += next->ChunkSize(); Used_ += Current->Used(); Wasted_ += Current->Wasted(); Current->Next = next; @@ -214,14 +221,18 @@ namespace NKikimr::NUtil { void DoRollback(TChunk* target, char* ptr) { if (target != Current) { TChunk* chunk = target; + size_t nextUsed = chunk->Used(); + size_t nextWasted = chunk->Wasted(); do { // Remove previously added stats - Used_ -= chunk->Used(); - Wasted_ -= chunk->Wasted(); + Used_ -= nextUsed; + Wasted_ -= nextWasted; // Switch to the next chunk in the chain chunk = chunk->Next; Y_VERIFY(chunk, "Rollback cannot find current chunk in the chain"); // Reset chunk and add it to stats as wasted/free space + nextUsed = chunk->Used(); + nextWasted = chunk->Wasted(); chunk->Reset(); Wasted_ += chunk->ChunkSize(); Available_ += chunk->Left; @@ -260,6 +271,7 @@ namespace NKikimr::NUtil { private: TChunk* First; TChunk* Current; + size_t Total_ = 0; size_t Used_ = 0; size_t Wasted_ = 0; size_t Available_ = 0; diff --git a/ydb/core/tablet_flat/util_pool_ut.cpp b/ydb/core/tablet_flat/util_pool_ut.cpp index 5f21e7af072..bb7a8502166 100644 --- a/ydb/core/tablet_flat/util_pool_ut.cpp +++ b/ydb/core/tablet_flat/util_pool_ut.cpp @@ -68,6 +68,25 @@ Y_UNIT_TEST_SUITE(TMemoryPoolTest) { DoTransactions(16); } + Y_UNIT_TEST(LongRollback) { + TMemoryPool pool(1); + UNIT_ASSERT_C(pool.Used() == 0, pool.Used()); + UNIT_ASSERT_C(pool.Used() + pool.Wasted() == pool.Total(), pool.Used() << " + " << pool.Wasted() << " != " << pool.Total()); + for (int j = 0; j < 2; ++j) { + pool.BeginTransaction(); + for (int i = 0; i < 1000; ++i) { + void* ptr = pool.Allocate(3); + UNIT_ASSERT(ptr); + } + UNIT_ASSERT_C(pool.Used() > 0, pool.Used()); + UNIT_ASSERT_C(pool.Wasted() < pool.Total(), pool.Wasted() << " >= " << pool.Total()); + UNIT_ASSERT_C(pool.Used() + pool.Wasted() == pool.Total(), pool.Used() << " + " << pool.Wasted() << " != " << pool.Total()); + pool.RollbackTransaction(); + UNIT_ASSERT_C(pool.Used() == 0, pool.Used()); + UNIT_ASSERT_C(pool.Used() + pool.Wasted() == pool.Total(), pool.Used() << " + " << pool.Wasted() << " != " << pool.Total()); + } + } + } // Y_UNIT_TEST_SUITE(TMemoryPoolTest) } // namespace NKikimr::NUtil |