aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorsnaury <snaury@ydb.tech>2023-02-09 11:23:12 +0300
committersnaury <snaury@ydb.tech>2023-02-09 11:23:12 +0300
commit407b9e52eea66d83508fb8abd1e44ae52afcd9b6 (patch)
treec1540d69056ce4be1f153f649fbba484acea994c
parentbbb79f5a581e9a3e2111f2823bcf16288e60e98c (diff)
downloadydb-407b9e52eea66d83508fb8abd1e44ae52afcd9b6.tar.gz
Fix memory pool stats recalculation on rollback
-rw-r--r--ydb/core/tablet_flat/CMakeLists.darwin.txt1
-rw-r--r--ydb/core/tablet_flat/CMakeLists.linux-aarch64.txt1
-rw-r--r--ydb/core/tablet_flat/CMakeLists.linux.txt1
-rw-r--r--ydb/core/tablet_flat/ut/CMakeLists.darwin.txt1
-rw-r--r--ydb/core/tablet_flat/ut/CMakeLists.linux-aarch64.txt1
-rw-r--r--ydb/core/tablet_flat/ut/CMakeLists.linux.txt1
-rw-r--r--ydb/core/tablet_flat/ut_util/CMakeLists.darwin.txt72
-rw-r--r--ydb/core/tablet_flat/ut_util/CMakeLists.linux-aarch64.txt75
-rw-r--r--ydb/core/tablet_flat/ut_util/CMakeLists.linux.txt77
-rw-r--r--ydb/core/tablet_flat/ut_util/CMakeLists.txt15
-rw-r--r--ydb/core/tablet_flat/util_pool.h18
-rw-r--r--ydb/core/tablet_flat/util_pool_ut.cpp19
12 files changed, 276 insertions, 6 deletions
diff --git a/ydb/core/tablet_flat/CMakeLists.darwin.txt b/ydb/core/tablet_flat/CMakeLists.darwin.txt
index 7398f2a9698..56e4ae1c32d 100644
--- a/ydb/core/tablet_flat/CMakeLists.darwin.txt
+++ b/ydb/core/tablet_flat/CMakeLists.darwin.txt
@@ -11,6 +11,7 @@ add_subdirectory(test)
add_subdirectory(ut)
add_subdirectory(ut_large)
add_subdirectory(ut_pg)
+add_subdirectory(ut_util)
add_library(ydb-core-tablet_flat)
target_compile_options(ydb-core-tablet_flat PRIVATE
diff --git a/ydb/core/tablet_flat/CMakeLists.linux-aarch64.txt b/ydb/core/tablet_flat/CMakeLists.linux-aarch64.txt
index 03ca64a9895..168a2df75a6 100644
--- a/ydb/core/tablet_flat/CMakeLists.linux-aarch64.txt
+++ b/ydb/core/tablet_flat/CMakeLists.linux-aarch64.txt
@@ -11,6 +11,7 @@ add_subdirectory(test)
add_subdirectory(ut)
add_subdirectory(ut_large)
add_subdirectory(ut_pg)
+add_subdirectory(ut_util)
add_library(ydb-core-tablet_flat)
target_compile_options(ydb-core-tablet_flat PRIVATE
diff --git a/ydb/core/tablet_flat/CMakeLists.linux.txt b/ydb/core/tablet_flat/CMakeLists.linux.txt
index 03ca64a9895..168a2df75a6 100644
--- a/ydb/core/tablet_flat/CMakeLists.linux.txt
+++ b/ydb/core/tablet_flat/CMakeLists.linux.txt
@@ -11,6 +11,7 @@ add_subdirectory(test)
add_subdirectory(ut)
add_subdirectory(ut_large)
add_subdirectory(ut_pg)
+add_subdirectory(ut_util)
add_library(ydb-core-tablet_flat)
target_compile_options(ydb-core-tablet_flat PRIVATE
diff --git a/ydb/core/tablet_flat/ut/CMakeLists.darwin.txt b/ydb/core/tablet_flat/ut/CMakeLists.darwin.txt
index cc9d15d8d84..c6b2d3446f4 100644
--- a/ydb/core/tablet_flat/ut/CMakeLists.darwin.txt
+++ b/ydb/core/tablet_flat/ut/CMakeLists.darwin.txt
@@ -48,7 +48,6 @@ target_sources(ydb-core-tablet_flat-ut PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_table_part_ut.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/flat_test_db.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/shared_handle_ut.cpp
- ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/util_pool_ut.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/ut_self.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/ut_iterator.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/ut_memtable.cpp
diff --git a/ydb/core/tablet_flat/ut/CMakeLists.linux-aarch64.txt b/ydb/core/tablet_flat/ut/CMakeLists.linux-aarch64.txt
index 175c70b919c..2cbc8909529 100644
--- a/ydb/core/tablet_flat/ut/CMakeLists.linux-aarch64.txt
+++ b/ydb/core/tablet_flat/ut/CMakeLists.linux-aarch64.txt
@@ -51,7 +51,6 @@ target_sources(ydb-core-tablet_flat-ut PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_table_part_ut.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/flat_test_db.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/shared_handle_ut.cpp
- ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/util_pool_ut.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/ut_self.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/ut_iterator.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/ut_memtable.cpp
diff --git a/ydb/core/tablet_flat/ut/CMakeLists.linux.txt b/ydb/core/tablet_flat/ut/CMakeLists.linux.txt
index cd630a668b7..e0a1415129e 100644
--- a/ydb/core/tablet_flat/ut/CMakeLists.linux.txt
+++ b/ydb/core/tablet_flat/ut/CMakeLists.linux.txt
@@ -53,7 +53,6 @@ target_sources(ydb-core-tablet_flat-ut PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_table_part_ut.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/flat_test_db.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/shared_handle_ut.cpp
- ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/util_pool_ut.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/ut_self.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/ut_iterator.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut/ut_memtable.cpp
diff --git a/ydb/core/tablet_flat/ut_util/CMakeLists.darwin.txt b/ydb/core/tablet_flat/ut_util/CMakeLists.darwin.txt
new file mode 100644
index 00000000000..375c1fa7820
--- /dev/null
+++ b/ydb/core/tablet_flat/ut_util/CMakeLists.darwin.txt
@@ -0,0 +1,72 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-core-tablet_flat-ut_util)
+target_include_directories(ydb-core-tablet_flat-ut_util PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat
+)
+target_link_libraries(ydb-core-tablet_flat-ut_util PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ ydb-core-tablet_flat
+)
+target_link_options(ydb-core-tablet_flat-ut_util PRIVATE
+ -Wl,-no_deduplicate
+ -Wl,-sdk_version,10.15
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ydb-core-tablet_flat-ut_util PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/util_pool_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-core-tablet_flat-ut_util
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-core-tablet_flat-ut_util
+ TEST_TARGET
+ ydb-core-tablet_flat-ut_util
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tablet_flat-ut_util
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tablet_flat-ut_util
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tablet_flat-ut_util
+ PROPERTY
+ TIMEOUT
+ 600
+)
+vcs_info(ydb-core-tablet_flat-ut_util)
diff --git a/ydb/core/tablet_flat/ut_util/CMakeLists.linux-aarch64.txt b/ydb/core/tablet_flat/ut_util/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..c2fba7805f6
--- /dev/null
+++ b/ydb/core/tablet_flat/ut_util/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,75 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-core-tablet_flat-ut_util)
+target_include_directories(ydb-core-tablet_flat-ut_util PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat
+)
+target_link_libraries(ydb-core-tablet_flat-ut_util PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-lfalloc
+ cpp-testing-unittest_main
+ ydb-core-tablet_flat
+)
+target_link_options(ydb-core-tablet_flat-ut_util PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-core-tablet_flat-ut_util PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/util_pool_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-core-tablet_flat-ut_util
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-core-tablet_flat-ut_util
+ TEST_TARGET
+ ydb-core-tablet_flat-ut_util
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tablet_flat-ut_util
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tablet_flat-ut_util
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tablet_flat-ut_util
+ PROPERTY
+ TIMEOUT
+ 600
+)
+vcs_info(ydb-core-tablet_flat-ut_util)
diff --git a/ydb/core/tablet_flat/ut_util/CMakeLists.linux.txt b/ydb/core/tablet_flat/ut_util/CMakeLists.linux.txt
new file mode 100644
index 00000000000..473fb6d1337
--- /dev/null
+++ b/ydb/core/tablet_flat/ut_util/CMakeLists.linux.txt
@@ -0,0 +1,77 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-core-tablet_flat-ut_util)
+target_include_directories(ydb-core-tablet_flat-ut_util PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat
+)
+target_link_libraries(ydb-core-tablet_flat-ut_util PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ ydb-core-tablet_flat
+)
+target_link_options(ydb-core-tablet_flat-ut_util PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-core-tablet_flat-ut_util PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/util_pool_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-core-tablet_flat-ut_util
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-core-tablet_flat-ut_util
+ TEST_TARGET
+ ydb-core-tablet_flat-ut_util
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tablet_flat-ut_util
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tablet_flat-ut_util
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-core-tablet_flat-ut_util
+ PROPERTY
+ TIMEOUT
+ 600
+)
+vcs_info(ydb-core-tablet_flat-ut_util)
diff --git a/ydb/core/tablet_flat/ut_util/CMakeLists.txt b/ydb/core/tablet_flat/ut_util/CMakeLists.txt
new file mode 100644
index 00000000000..5bb4faffb40
--- /dev/null
+++ b/ydb/core/tablet_flat/ut_util/CMakeLists.txt
@@ -0,0 +1,15 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND UNIX AND NOT APPLE AND NOT ANDROID)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin.txt)
+elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND UNIX AND NOT APPLE AND NOT ANDROID)
+ include(CMakeLists.linux.txt)
+endif()
diff --git a/ydb/core/tablet_flat/util_pool.h b/ydb/core/tablet_flat/util_pool.h
index e3cb17a5ee1..db6dbbc4103 100644
--- a/ydb/core/tablet_flat/util_pool.h
+++ b/ydb/core/tablet_flat/util_pool.h
@@ -111,7 +111,9 @@ namespace NKikimr::NUtil {
TMemoryPool(size_t initial)
: First(AllocateChunk(Max(initial, TChunk::ChunkHeaderSize() + PLATFORM_DATA_ALIGN)))
, Current(First)
- { }
+ {
+ Total_ = Current->ChunkSize();
+ }
~TMemoryPool() {
TChunk* chunk = First;
@@ -157,6 +159,10 @@ namespace NKikimr::NUtil {
RollbackState_.reset();
}
+ size_t Total() const noexcept {
+ return Total_;
+ }
+
size_t Used() const noexcept {
return Used_ + Current->Used();
}
@@ -190,6 +196,7 @@ namespace NKikimr::NUtil {
Y_VERIFY(!Current->Next);
size_t hint = Max(AlignUp<size_t>(sizeof(TChunk), PLATFORM_DATA_ALIGN) + size, Current->ChunkSize() + 1);
TChunk* next = AllocateChunk(hint);
+ Total_ += next->ChunkSize();
Used_ += Current->Used();
Wasted_ += Current->Wasted();
Current->Next = next;
@@ -214,14 +221,18 @@ namespace NKikimr::NUtil {
void DoRollback(TChunk* target, char* ptr) {
if (target != Current) {
TChunk* chunk = target;
+ size_t nextUsed = chunk->Used();
+ size_t nextWasted = chunk->Wasted();
do {
// Remove previously added stats
- Used_ -= chunk->Used();
- Wasted_ -= chunk->Wasted();
+ Used_ -= nextUsed;
+ Wasted_ -= nextWasted;
// Switch to the next chunk in the chain
chunk = chunk->Next;
Y_VERIFY(chunk, "Rollback cannot find current chunk in the chain");
// Reset chunk and add it to stats as wasted/free space
+ nextUsed = chunk->Used();
+ nextWasted = chunk->Wasted();
chunk->Reset();
Wasted_ += chunk->ChunkSize();
Available_ += chunk->Left;
@@ -260,6 +271,7 @@ namespace NKikimr::NUtil {
private:
TChunk* First;
TChunk* Current;
+ size_t Total_ = 0;
size_t Used_ = 0;
size_t Wasted_ = 0;
size_t Available_ = 0;
diff --git a/ydb/core/tablet_flat/util_pool_ut.cpp b/ydb/core/tablet_flat/util_pool_ut.cpp
index 5f21e7af072..bb7a8502166 100644
--- a/ydb/core/tablet_flat/util_pool_ut.cpp
+++ b/ydb/core/tablet_flat/util_pool_ut.cpp
@@ -68,6 +68,25 @@ Y_UNIT_TEST_SUITE(TMemoryPoolTest) {
DoTransactions(16);
}
+ Y_UNIT_TEST(LongRollback) {
+ TMemoryPool pool(1);
+ UNIT_ASSERT_C(pool.Used() == 0, pool.Used());
+ UNIT_ASSERT_C(pool.Used() + pool.Wasted() == pool.Total(), pool.Used() << " + " << pool.Wasted() << " != " << pool.Total());
+ for (int j = 0; j < 2; ++j) {
+ pool.BeginTransaction();
+ for (int i = 0; i < 1000; ++i) {
+ void* ptr = pool.Allocate(3);
+ UNIT_ASSERT(ptr);
+ }
+ UNIT_ASSERT_C(pool.Used() > 0, pool.Used());
+ UNIT_ASSERT_C(pool.Wasted() < pool.Total(), pool.Wasted() << " >= " << pool.Total());
+ UNIT_ASSERT_C(pool.Used() + pool.Wasted() == pool.Total(), pool.Used() << " + " << pool.Wasted() << " != " << pool.Total());
+ pool.RollbackTransaction();
+ UNIT_ASSERT_C(pool.Used() == 0, pool.Used());
+ UNIT_ASSERT_C(pool.Used() + pool.Wasted() == pool.Total(), pool.Used() << " + " << pool.Wasted() << " != " << pool.Total());
+ }
+ }
+
} // Y_UNIT_TEST_SUITE(TMemoryPoolTest)
} // namespace NKikimr::NUtil