Add large blobs data size

6 files changed, 129 insertions, 38 deletions

diff --git a/ydb/core/tablet_flat/flat_stat_part.h b/ydb/core/tablet_flat/flat_stat_part.h
index 3dd285c5cbf..dd00180a28a 100644
--- a/ydb/core/tablet_flat/flat_stat_part.h
+++ b/ydb/core/tablet_flat/flat_stat_part.h
@@ -118,11 +118,13 @@ private:
 // if page start key is not screened then the whole previous page is added to stats
 class TScreenedPartIndexIterator {
 public:
-    TScreenedPartIndexIterator(TPartView partView, TIntrusiveConstPtr<TKeyCellDefaults> keyColumns, TIntrusiveConstPtr<NPage::TFrames> small)
+    TScreenedPartIndexIterator(TPartView partView, TIntrusiveConstPtr<TKeyCellDefaults> keyColumns, 
+            TIntrusiveConstPtr<NPage::TFrames> small, TIntrusiveConstPtr<NPage::TFrames> large)
         : Part(std::move(partView.Part))
         , KeyColumns(std::move(keyColumns))
         , Screen(std::move(partView.Screen))
         , Small(std::move(small))
+        , Large(std::move(large))
         , CurrentHole(TScreen::Iter(Screen, CurrentHoleIdx, 0, 1))
     {
         Pos = Part->Index->Begin();
@@ -181,8 +183,13 @@ public:
             }
         }
 
-        if (rowCount && Small) {
-            AddSmallSize(stats.DataSize);
+        if (rowCount) {
+            if (Small) {
+                AddBlobsSize(stats.DataSize, Small.Get(), ELargeObj::Outer, PrevSmallPage);
+            }
+            if (Large) {
+                AddBlobsSize(stats.DataSize, Large.Get(), ELargeObj::Extern, PrevLargePage);
+            }
         }
 
         FillKey();
@@ -265,17 +272,17 @@ private:
     }
 
 private:
-    void AddSmallSize(TPartDataSize& stats) noexcept {
+    void AddBlobsSize(TPartDataSize& stats, const NPage::TFrames* frames, ELargeObj lob, ui32 &prevPage) noexcept {
         const auto row = GetLastRowId();
         const auto end = GetCurrentRowId();
 
-        PrevSmallPage = Small->Lower(row, PrevSmallPage, Max<ui32>());
+        prevPage = frames->Lower(row, prevPage, Max<ui32>());
 
-        while (auto &rel = Small->Relation(PrevSmallPage)) {
+        while (auto &rel = frames->Relation(prevPage)) {
             if (rel.Row < end) {
-                auto channel = Part->GetPageChannel(ELargeObj::Outer, PrevSmallPage);
+                auto channel = Part->GetPageChannel(lob, prevPage);
                 stats.Add(rel.Size, channel);
-                ++PrevSmallPage;
+                ++prevPage;
             } else if (!rel.IsHead()) {
                 Y_FAIL("Got unaligned NPage::TFrames head record");
             } else {
@@ -306,9 +313,11 @@ private:
     TSmallVec<TGroupState> HistoryGroups;
     TIntrusiveConstPtr<TScreen> Screen;
     TIntrusiveConstPtr<NPage::TFrames> Small;    /* Inverted index for small blobs   */
+    TIntrusiveConstPtr<NPage::TFrames> Large;    /* Inverted index for large blobs   */
     size_t CurrentHoleIdx = 0;
     TScreen::THole CurrentHole;
     ui32 PrevSmallPage = 0;
+    ui32 PrevLargePage = 0;
 };
 
 }}
diff --git a/ydb/core/tablet_flat/flat_stat_table.cpp b/ydb/core/tablet_flat/flat_stat_table.cpp
index b79568fa207..78db00d61b4 100644
--- a/ydb/core/tablet_flat/flat_stat_table.cpp
+++ b/ydb/core/tablet_flat/flat_stat_table.cpp
@@ -17,7 +17,7 @@ void BuildStats(const TSubset& subset, TStats& stats, ui64 rowCountResolution, u
     // Make index iterators for all parts
     for (auto& pi : subset.Flatten) {
         stats.IndexSize.Add(pi->IndexesRawSize, pi->Label.Channel());
-        TAutoPtr<TScreenedPartIndexIterator> iter = new TScreenedPartIndexIterator(pi, subset.Scheme->Keys, pi->Small);
+        TAutoPtr<TScreenedPartIndexIterator> iter = new TScreenedPartIndexIterator(pi, subset.Scheme->Keys, pi->Small, pi->Large);
         if (iter->IsValid()) {
             stIter.Add(iter);
         }
diff --git a/ydb/core/tablet_flat/flat_table_part_ut.cpp b/ydb/core/tablet_flat/flat_table_part_ut.cpp
index 9ebcd5c8077..ffb53ed8ab0 100644
--- a/ydb/core/tablet_flat/flat_table_part_ut.cpp
+++ b/ydb/core/tablet_flat/flat_table_part_ut.cpp
@@ -77,7 +77,7 @@ Y_UNIT_TEST_SUITE(TLegacy) {
         {
             TPartDataStats stats = { };
             // TScreenedPartIndexIterator without screen previously was TPartIndexIterator
-            TScreenedPartIndexIterator idxIter(TPartView{part, nullptr, nullptr}, scheme->Keys, nullptr);
+            TScreenedPartIndexIterator idxIter(TPartView{part, nullptr, nullptr}, scheme->Keys, nullptr, nullptr);
             sizes.clear();
 
             while (idxIter.IsValid()) {
@@ -144,7 +144,7 @@ Y_UNIT_TEST_SUITE(TLegacy) {
         auto fnIterate = [&dbgOut, &typeRegistry] (TIntrusiveConstPtr<TPartStore> part, TIntrusiveConstPtr<TScreen> screen,
                             TIntrusiveConstPtr<TRowScheme> scheme, TIntrusiveConstPtr<NPage::TFrames> frames) -> std::pair<ui64, ui64> {
             TPartDataStats stats = { };
-            TScreenedPartIndexIterator idxIter(TPartView{part, screen, nullptr}, scheme->Keys, std::move(frames));
+            TScreenedPartIndexIterator idxIter(TPartView{part, screen, nullptr}, scheme->Keys, std::move(frames), nullptr);
 
             while (idxIter.IsValid()) {
                 TDbTupleRef key = idxIter.GetCurrentKey();
@@ -302,8 +302,8 @@ Y_UNIT_TEST_SUITE(TLegacy) {
 
         TPartDataStats stats = { };
         TStatsIterator stIter(lay2.RowScheme()->Keys);
-        stIter.Add(MakeHolder<TScreenedPartIndexIterator>(TPartView{eggs2.At(0), screen2, nullptr}, lay2.RowScheme()->Keys, nullptr));
-        stIter.Add(MakeHolder<TScreenedPartIndexIterator>(TPartView{eggs1.At(0), screen1, nullptr}, lay2.RowScheme()->Keys, nullptr));
+        stIter.Add(MakeHolder<TScreenedPartIndexIterator>(TPartView{eggs2.At(0), screen2, nullptr}, lay2.RowScheme()->Keys, nullptr, nullptr));
+        stIter.Add(MakeHolder<TScreenedPartIndexIterator>(TPartView{eggs1.At(0), screen1, nullptr}, lay2.RowScheme()->Keys, nullptr, nullptr));
 
 
 
diff --git a/ydb/core/tx/datashard/datashard_ut_common.cpp b/ydb/core/tx/datashard/datashard_ut_common.cpp
index 7ae46deea40..3401582d574 100644
--- a/ydb/core/tx/datashard/datashard_ut_common.cpp
+++ b/ydb/core/tx/datashard/datashard_ut_common.cpp
@@ -1137,6 +1137,9 @@ void CreateShardedTable(
         if (family.LogPoolKind) fam->MutableStorageConfig()->MutableLog()->SetPreferredPoolKind(family.LogPoolKind);
         if (family.SysLogPoolKind) fam->MutableStorageConfig()->MutableSysLog()->SetPreferredPoolKind(family.SysLogPoolKind);
         if (family.DataPoolKind) fam->MutableStorageConfig()->MutableData()->SetPreferredPoolKind(family.DataPoolKind);
+        if (family.ExternalPoolKind) fam->MutableStorageConfig()->MutableExternal()->SetPreferredPoolKind(family.ExternalPoolKind);
+        if (family.DataThreshold) fam->MutableStorageConfig()->SetDataThreshold(family.DataThreshold);
+        if (family.ExternalThreshold) fam->MutableStorageConfig()->SetExternalThreshold(family.ExternalThreshold);
     }
 
     for (const auto& index : opts.Indexes_) {
diff --git a/ydb/core/tx/datashard/datashard_ut_common.h b/ydb/core/tx/datashard/datashard_ut_common.h
index 1aad79ce936..bf5772c93d5 100644
--- a/ydb/core/tx/datashard/datashard_ut_common.h
+++ b/ydb/core/tx/datashard/datashard_ut_common.h
@@ -446,6 +446,9 @@ struct TShardedTableOptions {
         TString LogPoolKind;
         TString SysLogPoolKind;
         TString DataPoolKind;
+        TString ExternalPoolKind;
+        ui64 DataThreshold = 0;
+        ui64 ExternalThreshold = 0;
     };
 
     using TAttributes = THashMap<TString, TString>;
diff --git a/ydb/core/tx/datashard/datashard_ut_stats.cpp b/ydb/core/tx/datashard/datashard_ut_stats.cpp
index e55dacc2d91..c1f77ee246f 100644
--- a/ydb/core/tx/datashard/datashard_ut_stats.cpp
+++ b/ydb/core/tx/datashard/datashard_ut_stats.cpp
@@ -99,7 +99,8 @@ Y_UNIT_TEST_SUITE(DataShardStats) {
             UNIT_ASSERT_VALUES_EQUAL(stats.GetDatashardId(), shards.at(0));
             UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetRowCount(), 3u);
             UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetPartCount(), 0u);
-            UNIT_ASSERT_GT(stats.GetTableStats().GetDataSize(), 0u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetDataSize(), 704u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetIndexSize(), 0u);
             pathId = TPathId(stats.GetTableOwnerId(), stats.GetTableLocalId());
         }
 
@@ -111,13 +112,13 @@ Y_UNIT_TEST_SUITE(DataShardStats) {
             UNIT_ASSERT_VALUES_EQUAL(stats.GetDatashardId(), shards.at(0));
             UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetRowCount(), 3u);
             UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetPartCount(), 1u);
-            UNIT_ASSERT_GT(stats.GetTableStats().GetDataSize(), 0u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetDataSize(), 65u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetIndexSize(), 54u);
 
             UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels().size(), 1);
-            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetDataSize(), 
-                stats.GetTableStats().GetDataSize() - stats.GetTableStats().GetInMemSize());
-            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetIndexSize(), 
-                stats.GetTableStats().GetIndexSize());
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetChannel(), 1u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetDataSize(), 65u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetIndexSize(), 54u);
         }
     }
 
@@ -155,7 +156,8 @@ Y_UNIT_TEST_SUITE(DataShardStats) {
             UNIT_ASSERT_VALUES_EQUAL(stats.GetDatashardId(), shards.at(0));
             UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetRowCount(), 3u);
             UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetPartCount(), 0u);
-            UNIT_ASSERT_GT(stats.GetTableStats().GetDataSize(), 0u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetDataSize(), 752u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetIndexSize(), 0u);
             pathId = TPathId(stats.GetTableOwnerId(), stats.GetTableLocalId());
         }
 
@@ -167,19 +169,16 @@ Y_UNIT_TEST_SUITE(DataShardStats) {
             UNIT_ASSERT_VALUES_EQUAL(stats.GetDatashardId(), shards.at(0));
             UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetRowCount(), 3u);
             UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetPartCount(), 1u);
-            UNIT_ASSERT_GT(stats.GetTableStats().GetDataSize(), 0u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetDataSize(), 115u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetIndexSize(), 82u);
 
             UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels().size(), 2);
-            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetChannel(), 1u);
-            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[1].GetChannel(), 2u);
-            
-            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetDataSize() + stats.GetTableStats().GetChannels()[1].GetDataSize(), 
-                stats.GetTableStats().GetDataSize() - stats.GetTableStats().GetInMemSize());
-            UNIT_ASSERT_GT(stats.GetTableStats().GetChannels()[0].GetDataSize(), 0u);
-            UNIT_ASSERT_GT(stats.GetTableStats().GetChannels()[1].GetDataSize(), 0u);
-
-            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetIndexSize(), stats.GetTableStats().GetIndexSize());
-            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[1].GetIndexSize(), 0);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetChannel(), 1u); // ssd
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetDataSize(), 65u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetIndexSize(), 82u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[1].GetChannel(), 2u); // hdd
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[1].GetDataSize(), 50u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[1].GetIndexSize(), 0u);
         }
     }
 
@@ -222,7 +221,8 @@ Y_UNIT_TEST_SUITE(DataShardStats) {
             UNIT_ASSERT_VALUES_EQUAL(stats.GetDatashardId(), shards.at(0));
             UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetRowCount(), count);
             UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetPartCount(), 0u);
-            UNIT_ASSERT_GT(stats.GetTableStats().GetDataSize(), 0u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetDataSize(), 196096u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetIndexSize(), 0u);
             pathId = TPathId(stats.GetTableOwnerId(), stats.GetTableLocalId());
         }
 
@@ -234,13 +234,12 @@ Y_UNIT_TEST_SUITE(DataShardStats) {
             UNIT_ASSERT_VALUES_EQUAL(stats.GetDatashardId(), shards.at(0));
             UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetRowCount(), count);
             UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetPartCount(), 1u);
-            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetDataSize(), 30100);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetDataSize(), 30100u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetIndexSize(), 138u);
 
-            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels().size(), 1);
-            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetDataSize(), 
-                stats.GetTableStats().GetDataSize() - stats.GetTableStats().GetInMemSize());
-            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetIndexSize(), 
-                stats.GetTableStats().GetIndexSize());
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetChannel(), 1u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetDataSize(), 30100u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetIndexSize(), 138u);
         }
 
         {
@@ -259,6 +258,83 @@ Y_UNIT_TEST_SUITE(DataShardStats) {
         NDataShard::gDbStatsRowCountResolution = gDbStatsRowCountResolutionBefore;
     }
 
+    Y_UNIT_TEST(BlobsStatsCorrect) {
+        TPortManager pm;
+        TServerSettings serverSettings(pm.GetPort(2134));
+        serverSettings.SetDomainName("Root")
+            .SetUseRealThreads(false)
+            .AddStoragePool("ssd")
+            .AddStoragePool("hdd")
+            .AddStoragePool("ext");
+
+        TServer::TPtr server = new TServer(serverSettings);
+        auto& runtime = *server->GetRuntime();
+        auto sender = runtime.AllocateEdgeActor();
+
+        runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
+
+        InitRoot(server, sender);
+
+        auto opts = TShardedTableOptions()
+            .Shards(1)
+            .Columns({
+                {"key", "Uint32", true, false}, 
+                {"value", "String", false, false}, 
+                {"value2", "String", false, false, "hdd"}})
+            .Families({
+                {.Name = "default", .LogPoolKind = "ssd", .SysLogPoolKind = "ssd", .DataPoolKind = "ssd", 
+                    .ExternalPoolKind = "ext", .DataThreshold = 100u, .ExternalThreshold = 200u}, 
+                {.Name = "hdd", .DataPoolKind = "hdd"}});
+        CreateShardedTable(server, sender, "/Root", "table-1", opts);
+
+        auto shards = GetTableShards(server, sender, "/Root/table-1");
+        UNIT_ASSERT_VALUES_EQUAL(shards.size(), 1u);
+
+        TString smallValue(150, 'S');
+        TString largeValue(1500, 'L');
+        ExecSQL(server, sender, (TString)"UPSERT INTO `/Root/table-1` (key, value, value2) VALUES " + 
+            "(1, \"AAA\", \"AAA\"), " + 
+            "(2, \"" + smallValue + "\", \"BBB\"), " + 
+            "(3, \"CCC\", \"" + smallValue + "\"), " + 
+            "(4, \"" + largeValue + "\", \"BBB\"), " + 
+            "(5, \"CCC\", \"" + largeValue + "\")");
+
+        TPathId pathId;
+        {
+            Cerr << "... waiting for stats after upsert" << Endl;
+            auto stats = WaitTableStats(runtime);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetDatashardId(), shards.at(0));
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetRowCount(), 5u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetPartCount(), 0u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetDataSize(), 4232u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetIndexSize(), 0u);
+            pathId = TPathId(stats.GetTableOwnerId(), stats.GetTableLocalId());
+        }
+
+        CompactTable(runtime, shards.at(0), pathId);
+
+        {
+            Cerr << "... waiting for stats after compaction" << Endl;
+            auto stats = WaitTableStats(runtime, /* minPartCount */ 1);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetDatashardId(), shards.at(0));
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetRowCount(), 5u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetPartCount(), 1u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetDataSize(), 3555u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetIndexSize(), 82u);
+
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels().size(), 3);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetChannel(), 1u); // ssd
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetDataSize(), 440u); // two small values
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetIndexSize(), 82u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[2].GetChannel(), 3u); // hdd
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[2].GetDataSize(), 99u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[2].GetIndexSize(), 0u);
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[1].GetChannel(), 2u); // ext
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[1].GetDataSize(), 3016u); // two large values
+            UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[1].GetIndexSize(), 0u);
+        }
+    }
+
 } // Y_UNIT_TEST_SUITE(DataShardStats)
 
 } // namespace NKikimr