Obtain UsedTableMemory from private cache stats, include historic indexes size (#6178)

7 files changed, 86 insertions, 36 deletions

diff --git a/ydb/core/tablet_flat/flat_executor.cpp b/ydb/core/tablet_flat/flat_executor.cpp
index 8d73dd4cc04..81bd63b86c7 100644
--- a/ydb/core/tablet_flat/flat_executor.cpp
+++ b/ydb/core/tablet_flat/flat_executor.cpp
@@ -3556,23 +3556,23 @@ void TExecutor::Handle(NOps::TEvResult *ops, TProdCompact *msg, bool cancelled)
 }
 
 void TExecutor::UpdateUsedTabletMemory() {
-    UsedTabletMemory = 0;
-    // Estimate memory usage for internal executor structures.
-    UsedTabletMemory += 50 << 10; // 50kb
-    // Count the number of bytes exclusive to private cache.
+    // Estimate memory usage for internal executor structures:
+    UsedTabletMemory = 50 << 10; // 50kb
+
+    // Count the number of bytes kept in private cache (can't be offloaded right now):
     if (PrivatePageCache) {
-        UsedTabletMemory += PrivatePageCache->GetStats().TotalExclusive;
+        UsedTabletMemory += PrivatePageCache->GetStats().TotalPinnedBody;
+        UsedTabletMemory += PrivatePageCache->GetStats().PinnedLoadSize;
     }
-    // Estimate memory used by database structures.
+
+    // Estimate memory used by internal database structures:
     auto &counters = Database->Counters();
     UsedTabletMemory += counters.MemTableWaste;
     UsedTabletMemory += counters.MemTableBytes;
-    UsedTabletMemory += counters.Parts.IndexBytes;
     UsedTabletMemory += counters.Parts.OtherBytes;
-    UsedTabletMemory += counters.Parts.ByKeyBytes;
     UsedTabletMemory += Stats->PacksMetaBytes;
 
-    // Add tablet memory usage.
+    // Add tablet memory usage:
     UsedTabletMemory += Owner->GetMemoryUsage();
 }
 
diff --git a/ydb/core/tablet_flat/flat_executor_ut.cpp b/ydb/core/tablet_flat/flat_executor_ut.cpp
index 5f9552f6f83..ab4dce353b5 100644
--- a/ydb/core/tablet_flat/flat_executor_ut.cpp
+++ b/ydb/core/tablet_flat/flat_executor_ut.cpp
@@ -6305,6 +6305,45 @@ Y_UNIT_TEST_SUITE(TFlatTableExecutorBTreeIndex) {
         UNIT_ASSERT_VALUES_EQUAL(failedAttempts, 330);
     }
 
+    Y_UNIT_TEST(EnableLocalDBBtreeIndex_False) { // uses flat index
+        TMyEnvBase env;
+        TRowsModel rows;
+
+        auto &appData = env->GetAppData();
+        
+        appData.FeatureFlags.SetEnableLocalDBBtreeIndex(false);
+        auto counters = MakeIntrusive<TSharedPageCacheCounters>(env->GetDynamicCounters());
+        int readRows = 0, failedAttempts = 0;
+
+        env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp));
+
+        auto policy = MakeIntrusive<TCompactionPolicy>();
+        policy->MinBTreeIndexNodeSize = 128;
+        env.SendSync(rows.MakeScheme(std::move(policy)));
+
+        env.SendSync(rows.VersionTo(TRowVersion(1, 10)).RowTo(0).MakeRows(1000, 950));
+        env.SendSync(rows.VersionTo(TRowVersion(2, 20)).RowTo(0).MakeRows(1000, 950));
+        
+        env.SendSync(new NFake::TEvCompact(TRowsModel::TableId));
+        env.WaitFor<NFake::TEvCompacted>();
+
+        // all pages are always kept in shared cache
+        UNIT_ASSERT_VALUES_EQUAL(counters->ActivePages->Val(), 290);
+
+        env.SendSync(new NFake::TEvExecute{ new TTxFullScan(readRows, failedAttempts) });
+        UNIT_ASSERT_VALUES_EQUAL(readRows, 1000);
+        UNIT_ASSERT_VALUES_EQUAL(failedAttempts, 0);
+
+        // restart tablet
+        env.SendSync(new TEvents::TEvPoison, false, true);
+        env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp));
+
+        // after restart we have no pages in private cache
+        env.SendSync(new NFake::TEvExecute{ new TTxFullScan(readRows, failedAttempts) }, true);
+        UNIT_ASSERT_VALUES_EQUAL(readRows, 1000);
+        UNIT_ASSERT_VALUES_EQUAL(failedAttempts, 288);
+    }
+
     Y_UNIT_TEST(EnableLocalDBBtreeIndex_True_EnableLocalDBFlatIndex_False) { // uses b-tree index
         TMyEnvBase env;
         TRowsModel rows;
diff --git a/ydb/core/tablet_flat/flat_part_loader.cpp b/ydb/core/tablet_flat/flat_part_loader.cpp
index b031c698cce..33c69b131c8 100644
--- a/ydb/core/tablet_flat/flat_part_loader.cpp
+++ b/ydb/core/tablet_flat/flat_part_loader.cpp
@@ -210,17 +210,22 @@ TAutoPtr<NPageCollection::TFetch> TLoader::StageCreatePartView() noexcept
     TEpoch epoch = Epoch != TEpoch::Max() ? Epoch : TEpoch(Root.GetEpoch());
 
     // TODO: put index size to stat?
-    // TODO: include history indexes bytes
     size_t indexesRawSize = 0;
     if (BTreeGroupIndexes) {
         for (const auto &meta : BTreeGroupIndexes) {
             indexesRawSize += meta.IndexSize;
         }
+        for (const auto &meta : BTreeHistoricIndexes) {
+            indexesRawSize += meta.IndexSize;
+        }
         // Note: although we also have flat index, it shouldn't be loaded; so let's not count it here
     } else {
         for (auto indexPage : FlatGroupIndexes) {
             indexesRawSize += Packs[0]->GetPageSize(indexPage);
         }
+        for (auto indexPage : FlatHistoricIndexes) {
+            indexesRawSize += Packs[0]->GetPageSize(indexPage);
+        }
     }
 
     auto *partStore = new TPartStore(
diff --git a/ydb/core/tablet_flat/flat_stat_table_btree_index_histogram.cpp b/ydb/core/tablet_flat/flat_stat_table_btree_index_histogram.cpp
index 3a8dbfd483d..4c060d59478 100644
--- a/ydb/core/tablet_flat/flat_stat_table_btree_index_histogram.cpp
+++ b/ydb/core/tablet_flat/flat_stat_table_btree_index_histogram.cpp
@@ -150,7 +150,7 @@ class TTableHistogramBuilderBtreeIndex {
 
             // category = -1 for Key = { }, IsBegin = true
             // category =  0 for Key = {*}, IsBegin = *
-            // category = -1 for Key = { }, IsBegin = false
+            // category = +1 for Key = { }, IsBegin = false
             return Compare(GetCategory(a), GetCategory(b));
         }
 
@@ -159,12 +159,12 @@ class TTableHistogramBuilderBtreeIndex {
             if (a.Key) {
                 return 0;
             }
-            return a.IsBegin ? -1 : 1;
+            return a.IsBegin ? -1 : +1;
         }
 
         static i8 Compare(i8 a, i8 b) noexcept {
             if (a < b) return -1;
-            if (a > b) return 1;
+            if (a > b) return +1;
             return 0;
         }
     };
diff --git a/ydb/core/tablet_flat/test/libs/table/test_store.h b/ydb/core/tablet_flat/test/libs/table/test_store.h
index e699fa92e81..761d5c939cb 100644
--- a/ydb/core/tablet_flat/test/libs/table/test_store.h
+++ b/ydb/core/tablet_flat/test/libs/table/test_store.h
@@ -24,8 +24,8 @@ namespace NTest {
 
         struct TEggs {
             bool Rooted;
-            TVector<TPageId> GroupIndexes;
-            TVector<TPageId> HistoricIndexes;
+            TVector<TPageId> FlatGroupIndexes;
+            TVector<TPageId> FlatHistoricIndexes;
             TVector<NPage::TBtreeIndexMeta> BTreeGroupIndexes;
             TVector<NPage::TBtreeIndexMeta> BTreeHistoricIndexes;
             TData *Scheme;
diff --git a/ydb/core/tablet_flat/test/libs/table/test_writer.h b/ydb/core/tablet_flat/test/libs/table/test_writer.h
index 70e916d7b65..f710071fb6c 100644
--- a/ydb/core/tablet_flat/test/libs/table/test_writer.h
+++ b/ydb/core/tablet_flat/test/libs/table/test_writer.h
@@ -77,8 +77,14 @@ namespace NTest {
                 for (const auto &meta : eggs.BTreeGroupIndexes) {
                     indexesRawSize += meta.IndexSize;
                 }
+                for (const auto &meta : eggs.BTreeHistoricIndexes) {
+                    indexesRawSize += meta.IndexSize;
+                }
             } else {
-                for (auto indexPage : eggs.GroupIndexes) {
+                for (auto indexPage : eggs.FlatGroupIndexes) {
+                    indexesRawSize += Store->GetPageSize(0, indexPage);
+                }
+                for (auto indexPage : eggs.FlatHistoricIndexes) {
                     indexesRawSize += Store->GetPageSize(0, indexPage);
                 }
             }
@@ -90,7 +96,7 @@ namespace NTest {
                     {
                         epoch,
                         TPartScheme::Parse(*eggs.Scheme, eggs.Rooted),
-                        { eggs.GroupIndexes, eggs.HistoricIndexes, eggs.BTreeGroupIndexes, eggs.BTreeHistoricIndexes },
+                        { eggs.FlatGroupIndexes, eggs.FlatHistoricIndexes, eggs.BTreeGroupIndexes, eggs.BTreeHistoricIndexes },
                         eggs.Blobs ? new TExtBlobs(*eggs.Blobs, { }) : nullptr,
                         eggs.ByKey ? new TBloom(*eggs.ByKey) : nullptr,
                         eggs.Large ? new TFrames(*eggs.Large) : nullptr,
diff --git a/ydb/core/tablet_flat/ut/ut_stat.cpp b/ydb/core/tablet_flat/ut/ut_stat.cpp
index f17e1bfea9b..ec265efe8da 100644
--- a/ydb/core/tablet_flat/ut/ut_stat.cpp
+++ b/ydb/core/tablet_flat/ut/ut_stat.cpp
@@ -143,14 +143,14 @@ Y_UNIT_TEST_SUITE(BuildStatsFlatIndex) {
     Y_UNIT_TEST(Single_History)
     {
         auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);   
-        CheckMixedIndex(*subset, 24000, 3547100, 31242);
+        CheckMixedIndex(*subset, 24000, 3547100, 49916);
     }
 
     Y_UNIT_TEST(Single_History_Slices)
     {
         auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);   
         subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
-        CheckMixedIndex(*subset, 9582, 1425198, 31242);
+        CheckMixedIndex(*subset, 9582, 1425198, 49916);
     }
 
     Y_UNIT_TEST(Single_Groups)
@@ -169,14 +169,14 @@ Y_UNIT_TEST_SUITE(BuildStatsFlatIndex) {
     Y_UNIT_TEST(Single_Groups_History)
     {
         auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);   
-        CheckMixedIndex(*subset, 24000, 4054050, 18810);
+        CheckMixedIndex(*subset, 24000, 4054050, 29361);
     }
 
     Y_UNIT_TEST(Single_Groups_History_Slices)
     {
         auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);   
         subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
-        CheckMixedIndex(*subset, 13570, 2277890, 18810);
+        CheckMixedIndex(*subset, 13570, 2277890, 29361);
     }
 
     Y_UNIT_TEST(Mixed)
@@ -194,7 +194,7 @@ Y_UNIT_TEST_SUITE(BuildStatsFlatIndex) {
     Y_UNIT_TEST(Mixed_Groups_History)
     {
         auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, TMixerRnd(4), 0.3);
-        CheckMixedIndex(*subset, 24000, 4054270, 19152);
+        CheckMixedIndex(*subset, 24000, 4054270, 29970);
     }
 
     Y_UNIT_TEST(Serial)
@@ -215,7 +215,7 @@ Y_UNIT_TEST_SUITE(BuildStatsFlatIndex) {
     {
         TMixerSeq mixer(4, Mass1.Saved.Size());
         auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, mixer, 0.3);
-        CheckMixedIndex(*subset, 24000, 4054290, 19168);
+        CheckMixedIndex(*subset, 24000, 4054290, 30013);
     }
 }
 
@@ -239,14 +239,14 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) {
     Y_UNIT_TEST(Single_History)
     {
         auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);
-        CheckMixedIndex(*subset, 24000, 3547100, 61162);
+        CheckMixedIndex(*subset, 24000, 3547100, 81694);
     }
 
     Y_UNIT_TEST(Single_History_Slices)
     {
         auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);   
         subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
-        CheckMixedIndex(*subset, 9582, 1425198, 61162);
+        CheckMixedIndex(*subset, 9582, 1425198, 81694);
     }
 
     Y_UNIT_TEST(Single_Groups)
@@ -265,14 +265,14 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) {
     Y_UNIT_TEST(Single_Groups_History)
     {
         auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);   
-        CheckMixedIndex(*subset, 24000, 4054050, 34837);
+        CheckMixedIndex(*subset, 24000, 4054050, 46562);
     }
 
     Y_UNIT_TEST(Single_Groups_History_Slices)
     {
         auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);   
         subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
-        CheckMixedIndex(*subset, 13570, 2277890, 34837);
+        CheckMixedIndex(*subset, 13570, 2277890, 46562);
     }
 
     Y_UNIT_TEST(Mixed)
@@ -290,7 +290,7 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) {
     Y_UNIT_TEST(Mixed_Groups_History)
     {
         auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, TMixerRnd(4), 0.3);
-        CheckMixedIndex(*subset, 24000, 4054270, 34579);
+        CheckMixedIndex(*subset, 24000, 4054270, 46543);
     }
 
     Y_UNIT_TEST(Serial)
@@ -311,7 +311,7 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) {
     {
         TMixerSeq mixer(4, Mass1.Saved.Size());
         auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, mixer, 0.3);
-        CheckMixedIndex(*subset, 24000, 4054290, 34652);
+        CheckMixedIndex(*subset, 24000, 4054290, 46640);
     }
 
     Y_UNIT_TEST(Single_LowResolution)
@@ -343,14 +343,14 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) {
     Y_UNIT_TEST(Single_Groups_History_LowResolution)
     {
         auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), true, WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);   
-        CheckMixedIndex(*subset, 24000, 4054050, 48540, 5310, 531050);
+        CheckMixedIndex(*subset, 24000, 4054050, 64742, 5310, 531050);
     }
 
     Y_UNIT_TEST(Single_Groups_History_Slices_LowResolution)
     {
         auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), true, WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);   
         subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
-        CheckMixedIndex(*subset, 13570, 2234982 /* ~2277890 */, 48540, 5310, 531050);
+        CheckMixedIndex(*subset, 13570, 2234982 /* ~2277890 */, 64742, 5310, 531050);
     }
 }
 
@@ -374,14 +374,14 @@ Y_UNIT_TEST_SUITE(BuildStatsBTreeIndex) {
     Y_UNIT_TEST(Single_History)
     {
         auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);
-        CheckBTreeIndex(*subset, 24000, 3547100, 61162);
+        CheckBTreeIndex(*subset, 24000, 3547100, 81694);
     }
 
     Y_UNIT_TEST(Single_History_Slices)
     {
         auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);   
         subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
-        CheckBTreeIndex(*subset, 9582, 1425282, 61162);
+        CheckBTreeIndex(*subset, 9582, 1425282, 81694);
     }
 
     Y_UNIT_TEST(Single_Groups)
@@ -400,14 +400,14 @@ Y_UNIT_TEST_SUITE(BuildStatsBTreeIndex) {
     Y_UNIT_TEST(Single_Groups_History)
     {
         auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);   
-        CheckBTreeIndex(*subset, 24000, 4054050, 34837);
+        CheckBTreeIndex(*subset, 24000, 4054050, 46562);
     }
 
     Y_UNIT_TEST(Single_Groups_History_Slices)
     {
         auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);   
         subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
-        CheckBTreeIndex(*subset, 13570, 2273213, 34837);
+        CheckBTreeIndex(*subset, 13570, 2273213, 46562);
     }
 
     Y_UNIT_TEST(Mixed)
@@ -425,7 +425,7 @@ Y_UNIT_TEST_SUITE(BuildStatsBTreeIndex) {
     Y_UNIT_TEST(Mixed_Groups_History)
     {
         auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, TMixerRnd(4), 0.3);
-        CheckBTreeIndex(*subset, 24000, 4054270, 34579);
+        CheckBTreeIndex(*subset, 24000, 4054270, 46543);
     }
 }