restore tests

8 files changed, 53 insertions, 22 deletions

diff --git a/ydb/core/protos/tx_columnshard.proto b/ydb/core/protos/tx_columnshard.proto
index 2b969628f61..1700172483f 100644
--- a/ydb/core/protos/tx_columnshard.proto
+++ b/ydb/core/protos/tx_columnshard.proto
@@ -36,7 +36,6 @@ message TReadStats {
     optional uint64 IndexBatches = 6;
     optional uint64 NotIndexedBatches = 7;
     optional uint32 SchemaColumns = 8;
-    optional uint64 PortionsBytes = 9;
     optional uint64 DataFilterBytes = 10;
     optional uint64 DataAdditionalBytes = 11;
 
@@ -44,6 +43,9 @@ message TReadStats {
     optional uint32 AdditionalColumns = 13;
 
     optional uint32 SelectedRows = 14;
+    optional uint64 CompactedPortionsBytes = 15;
+    optional uint64 InsertedPortionsBytes = 16;
+    optional uint64 CommittedPortionsBytes = 17;
 }
 
 message TLogicalMetadata {
diff --git a/ydb/core/tx/columnshard/engines/insert_table/data.h b/ydb/core/tx/columnshard/engines/insert_table/data.h
index 954e4e82e20..60361d2e27f 100644
--- a/ydb/core/tx/columnshard/engines/insert_table/data.h
+++ b/ydb/core/tx/columnshard/engines/insert_table/data.h
@@ -130,6 +130,10 @@ private:
     YDB_READONLY_DEF(std::optional<NArrow::TReplaceKey>, First);
     YDB_READONLY_DEF(std::optional<NArrow::TReplaceKey>, Last);
 public:
+    ui64 GetSize() const {
+        return BlobRange.Size;
+    }
+
     const NArrow::TReplaceKey& GetFirstVerified() const {
         Y_ABORT_UNLESS(First);
         return *First;
diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/plain_read_data.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/plain_read_data.cpp
index 7522a4a3cc1..d84609c623d 100644
--- a/ydb/core/tx/columnshard/engines/reader/plain_reader/plain_read_data.cpp
+++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/plain_read_data.cpp
@@ -12,7 +12,9 @@ TPlainReadData::TPlainReadData(const std::shared_ptr<NOlap::TReadContext>& conte
     const auto& committed = GetReadMetadata()->CommittedBlobs;
     auto itCommitted = committed.begin();
     auto itPortion = portionsOrdered.begin();
-    ui64 portionsBytes = 0;
+    ui64 committedPortionsBytes = 0;
+    ui64 insertedPortionsBytes = 0;
+    ui64 compactedPortionsBytes = 0;
     while (itCommitted != committed.end() || itPortion != portionsOrdered.end()) {
         bool movePortion = false;
         if (itCommitted == committed.end()) {
@@ -26,7 +28,11 @@ TPlainReadData::TPlainReadData(const std::shared_ptr<NOlap::TReadContext>& conte
         }
 
         if (movePortion) {
-            portionsBytes += (*itPortion)->BlobsBytes();
+            if ((*itPortion)->GetMeta().GetProduced() == NPortion::EProduced::COMPACTED || (*itPortion)->GetMeta().GetProduced() == NPortion::EProduced::SPLIT_COMPACTED) {
+                compactedPortionsBytes += (*itPortion)->BlobsBytes();
+            } else {
+                insertedPortionsBytes += (*itPortion)->BlobsBytes();
+            }
             auto start = GetReadMetadata()->BuildSortedPosition((*itPortion)->IndexKeyStart());
             auto finish = GetReadMetadata()->BuildSortedPosition((*itPortion)->IndexKeyEnd());
             AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "portions_for_merge")("start", start.DebugJson())("finish", finish.DebugJson());
@@ -36,6 +42,7 @@ TPlainReadData::TPlainReadData(const std::shared_ptr<NOlap::TReadContext>& conte
             auto start = GetReadMetadata()->BuildSortedPosition(itCommitted->GetFirstVerified());
             auto finish = GetReadMetadata()->BuildSortedPosition(itCommitted->GetLastVerified());
             sources.emplace_back(std::make_shared<TCommittedDataSource>(sourceIdx++, *itCommitted, SpecialReadContext, start, finish));
+            committedPortionsBytes += itCommitted->GetSize();
             ++itCommitted;
         }
     }
@@ -46,7 +53,9 @@ TPlainReadData::TPlainReadData(const std::shared_ptr<NOlap::TReadContext>& conte
     stats->IndexBatches = GetReadMetadata()->NumIndexedBlobs();
     stats->CommittedBatches = GetReadMetadata()->CommittedBlobs.size();
     stats->SchemaColumns = (*SpecialReadContext->GetProgramInputColumns() - *SpecialReadContext->GetSpecColumns()).GetSize();
-    stats->PortionsBytes = portionsBytes;
+    stats->CommittedPortionsBytes = committedPortionsBytes;
+    stats->InsertedPortionsBytes = insertedPortionsBytes;
+    stats->CompactedPortionsBytes = compactedPortionsBytes;
 
 }
 
diff --git a/ydb/core/tx/columnshard/engines/reader/read_metadata.cpp b/ydb/core/tx/columnshard/engines/reader/read_metadata.cpp
index 33e6b7f6cd5..f5474a7497d 100644
--- a/ydb/core/tx/columnshard/engines/reader/read_metadata.cpp
+++ b/ydb/core/tx/columnshard/engines/reader/read_metadata.cpp
@@ -94,10 +94,12 @@ void TReadStats::PrintToLog() {
         ("schema_columns", SchemaColumns)
         ("filter_columns", FilterColumns)
         ("additional_columns", AdditionalColumns)
-        ("portions_bytes", PortionsBytes)
+        ("compacted_portions_bytes", CompactedPortionsBytes)
+        ("inserted_portions_bytes", InsertedPortionsBytes)
+        ("committed_portions_bytes", CommittedPortionsBytes)
         ("data_filter_bytes", DataFilterBytes)
         ("data_additional_bytes", DataAdditionalBytes)
-        ("delta_bytes", PortionsBytes - DataFilterBytes - DataAdditionalBytes)
+        ("delta_bytes", CompactedPortionsBytes + InsertedPortionsBytes + CommittedPortionsBytes - DataFilterBytes - DataAdditionalBytes)
         ("selected_rows", SelectedRows)
         ;
 }
diff --git a/ydb/core/tx/columnshard/engines/reader/read_metadata.h b/ydb/core/tx/columnshard/engines/reader/read_metadata.h
index af6fa21bff9..a09a017ce2b 100644
--- a/ydb/core/tx/columnshard/engines/reader/read_metadata.h
+++ b/ydb/core/tx/columnshard/engines/reader/read_metadata.h
@@ -30,7 +30,9 @@ struct TReadStats {
     ui64 IndexPortions{0};
     ui64 IndexBatches{0};
     ui64 CommittedBatches{0};
-    ui64 PortionsBytes{ 0 };
+    ui64 CommittedPortionsBytes = 0;
+    ui64 InsertedPortionsBytes = 0;
+    ui64 CompactedPortionsBytes = 0;
     ui64 DataFilterBytes{ 0 };
     ui64 DataAdditionalBytes{ 0 };
 
diff --git a/ydb/core/tx/columnshard/read_actor.cpp b/ydb/core/tx/columnshard/read_actor.cpp
index d5afc8cd5fc..475c2a34267 100644
--- a/ydb/core/tx/columnshard/read_actor.cpp
+++ b/ydb/core/tx/columnshard/read_actor.cpp
@@ -127,7 +127,9 @@ public:
             protoStats->SetAdditionalColumns(stats->AdditionalColumns);
             protoStats->SetDataFilterBytes(stats->DataFilterBytes);
             protoStats->SetDataAdditionalBytes(stats->DataAdditionalBytes);
-            protoStats->SetPortionsBytes(stats->PortionsBytes);
+            protoStats->SetCompactedPortionsBytes(stats->CompactedPortionsBytes);
+            protoStats->SetInsertedPortionsBytes(stats->InsertedPortionsBytes);
+            protoStats->SetCommittedPortionsBytes(stats->CommittedPortionsBytes);
             protoStats->SetSelectedRows(stats->SelectedRows);
         }
 
diff --git a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp
index 1dee12340f6..97e883aa3c1 100644
--- a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp
+++ b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp
@@ -972,19 +972,29 @@ void TestWriteRead(bool reboots, const TestTableDescription& table = {}, TString
             auto& readStats = meta.GetReadStats();
 
             if (ydbSchema == TTestSchema::YdbSchema()) {
-                Cerr << codec << "/" << readStats.GetPortionsBytes() << Endl;
-//                if (codec == "" || codec == "lz4") {
-//                    UNIT_ASSERT_GE(readStats.GetPortionsBytes() / 100000, 40);
-//                    UNIT_ASSERT_LE(readStats.GetPortionsBytes() / 100000, 50);
-//                } else if (codec == "none") {
-//                    UNIT_ASSERT_GE(readStats.GetPortionsBytes() / 100000, 65);
-//                    UNIT_ASSERT_LE(readStats.GetPortionsBytes() / 100000, 78);
-//                } else if (codec == "zstd") {
-//                    UNIT_ASSERT_GE(readStats.GetPortionsBytes() / 100000, 20);
-//                    UNIT_ASSERT_LE(readStats.GetPortionsBytes() / 100000, 30);
-//                } else {
-//                    UNIT_ASSERT(false);
-//                }
+                Cerr << codec << "/" << readStats.GetCompactedPortionsBytes() << "/" << readStats.GetInsertedPortionsBytes() << "/" << readStats.GetCommittedPortionsBytes() << Endl;
+                if (readStats.GetInsertedPortionsBytes()) {
+                    UNIT_ASSERT_GE(readStats.GetInsertedPortionsBytes() / 100000, 40);
+                    UNIT_ASSERT_LE(readStats.GetInsertedPortionsBytes() / 100000, 50);
+                }
+                if (readStats.GetCommittedPortionsBytes()) {
+                    UNIT_ASSERT_GE(readStats.GetCommittedPortionsBytes() / 100000, 65);
+                    UNIT_ASSERT_LE(readStats.GetCommittedPortionsBytes() / 100000, 78);
+                }
+                if (readStats.GetCompactedPortionsBytes()) {
+                    if (codec == "" || codec == "lz4") {
+                        UNIT_ASSERT_GE(readStats.GetCompactedPortionsBytes() / 100000, 40);
+                        UNIT_ASSERT_LE(readStats.GetCompactedPortionsBytes() / 100000, 50);
+                    } else if (codec == "none") {
+                        UNIT_ASSERT_GE(readStats.GetCompactedPortionsBytes() / 100000, 65);
+                        UNIT_ASSERT_LE(readStats.GetCompactedPortionsBytes() / 100000, 78);
+                    } else if (codec == "zstd") {
+                        UNIT_ASSERT_GE(readStats.GetCompactedPortionsBytes() / 100000, 20);
+                        UNIT_ASSERT_LE(readStats.GetCompactedPortionsBytes() / 100000, 30);
+                    } else {
+                        UNIT_ASSERT(false);
+                    }
+                }
             }
         }
     }
diff --git a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp
index 11fe7c78731..164f117238d 100644
--- a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp
+++ b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp
@@ -844,7 +844,7 @@ std::vector<std::pair<ui32, ui64>> TestTiers(bool reboots, const std::vector<TSt
             if (resRead.GetFinished()) {
                 UNIT_ASSERT(meta.HasReadStats());
                 auto& readStats = meta.GetReadStats();
-                ui64 numBytes = readStats.GetPortionsBytes(); // compressed bytes in storage
+                const ui64 numBytes = readStats.GetCompactedPortionsBytes() + readStats.GetInsertedPortionsBytes() + readStats.GetCommittedPortionsBytes();
                 specRowsBytes.back().second += numBytes;
                 numExpected = resRead.GetBatch() + 1;
             }