Add InternalPathId column to primary_index_granule_stats (#19644)

7 files changed, 117 insertions, 5 deletions

diff --git a/ydb/core/kqp/ut/olap/helpers/local.h b/ydb/core/kqp/ut/olap/helpers/local.h
index 0ac389f9944..1e04b4ca4c7 100644
--- a/ydb/core/kqp/ut/olap/helpers/local.h
+++ b/ydb/core/kqp/ut/olap/helpers/local.h
@@ -36,7 +36,7 @@ public:
         CreateOlapTablesWithStore(tableNames, storeName, storeShardsCount, tableShardsCount);
     }
 
-    void CreateTestOlapTableWithoutStore(TString tableName = "olapTable", ui32 tableShardsCount = 3) {
+    void CreateTestOlapStandaloneTable(TString tableName = "olapTable", ui32 tableShardsCount = 3) {
         CreateOlapTables({tableName}, tableShardsCount);
     }
 
diff --git a/ydb/core/kqp/ut/olap/indexes_ut.cpp b/ydb/core/kqp/ut/olap/indexes_ut.cpp
index 8152ccba8c8..5c9227d5adf 100644
--- a/ydb/core/kqp/ut/olap/indexes_ut.cpp
+++ b/ydb/core/kqp/ut/olap/indexes_ut.cpp
@@ -156,7 +156,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) {
         csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1));
         csController->SetOverrideBlobSplitSettings(NOlap::NSplitter::TSplitSettings());
 
-        TLocalHelper(kikimr).CreateTestOlapTableWithoutStore();
+        TLocalHelper(kikimr).CreateTestOlapStandaloneTable();
         auto tableClient = kikimr.GetTableClient();
         auto& client = kikimr.GetTestClient();
 
diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp
index 6d034957aa1..aca1d2c6200 100644
--- a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp
+++ b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp
@@ -220,7 +220,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) {
         auto settings = TKikimrSettings()
              .SetWithSampleTables(false);
         TKikimrRunner kikimr(settings);
-        TLocalHelper(kikimr).CreateTestOlapTableWithoutStore();
+        TLocalHelper(kikimr).CreateTestOlapStandaloneTable();
 
         {
             //1. QueryService
diff --git a/ydb/core/kqp/ut/olap/sys_view_ut.cpp b/ydb/core/kqp/ut/olap/sys_view_ut.cpp
index d74c64b227c..46e9d7f78e3 100644
--- a/ydb/core/kqp/ut/olap/sys_view_ut.cpp
+++ b/ydb/core/kqp/ut/olap/sys_view_ut.cpp
@@ -12,6 +12,115 @@
 namespace NKikimr::NKqp {
 
 Y_UNIT_TEST_SUITE(KqpOlapSysView) {
+
+
+    Y_UNIT_TEST(GranulePathId_Store) {
+        auto settings = TKikimrSettings().SetWithSampleTables(false);
+        TKikimrRunner kikimr(settings);
+
+        auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>();
+        auto helper = TLocalHelper(kikimr);
+        const ui32 storeShardsCount = 101;
+        const ui32 tableShardsCount = 17;
+        const ui32 tablesCount = 1013;
+        TVector<TString> tableNames;
+        for (ui32 i = 0; i < tablesCount; ++i) {
+            tableNames.push_back(TStringBuilder() << "table" << i);
+        }
+        helper.CreateTestOlapTables(tableNames, "columnStore", storeShardsCount, tableShardsCount);
+        const auto tablets = csController->GetActiveTablets();
+        UNIT_ASSERT_VALUES_EQUAL(tablets.size(), storeShardsCount);
+
+        auto tableClient = kikimr.GetTableClient();
+
+        {
+            //check the store
+            auto selectQuery = TString(R"(
+                SELECT PathId, TabletId, InternalPathId, 
+                FROM `/Root/columnStore/.sys/store_primary_index_granule_stats`
+            )");
+
+            auto rows = ExecuteScanQuery(tableClient, selectQuery, true);
+            UNIT_ASSERT_VALUES_EQUAL(rows.size(), tableShardsCount * tablesCount);
+            THashMap<NColumnShard::TSchemeShardLocalPathId, THashMap<ui64, NColumnShard::TInternalPathId>> result;
+            for (const auto& row : rows) {
+                result[NColumnShard::TSchemeShardLocalPathId::FromRawValue(GetUint64(row.at("PathId")))][GetUint64(row.at("TabletId"))] =
+                    NColumnShard::TInternalPathId::FromRawValue(GetUint64(row.at("InternalPathId")));
+            }
+            UNIT_ASSERT_VALUES_EQUAL(result.size(), tablesCount);
+
+            for (const auto& [tabletId, pathIdTranslator]  : tablets) {
+                const auto& pathIds = pathIdTranslator->GetSchemeShardLocalPathIds();
+                for (const auto& pathId : pathIds) {
+                    const auto& internalPathId = pathIdTranslator->ResolveInternalPathId(pathId);
+                    UNIT_ASSERT(internalPathId.has_value());
+                    UNIT_ASSERT(result.contains(pathId) && result[pathId].contains(tabletId));
+                    UNIT_ASSERT_VALUES_EQUAL(result[pathId][tabletId], *internalPathId);
+                }
+            }
+        }
+
+        {
+            //check a table in the store
+            auto selectQuery = TString(R"(
+                SELECT PathId, TabletId, InternalPathId, 
+                FROM `/Root/columnStore/table2/.sys/primary_index_granule_stats`
+            )");
+
+            auto rows = ExecuteScanQuery(tableClient, selectQuery, true);
+            UNIT_ASSERT_VALUES_EQUAL(rows.size(), tableShardsCount);
+            THashMap<NColumnShard::TSchemeShardLocalPathId, THashMap<ui64, NColumnShard::TInternalPathId>> result;
+            for (const auto& row : rows) {
+                result[NColumnShard::TSchemeShardLocalPathId::FromRawValue(GetUint64(row.at("PathId")))][GetUint64(row.at("TabletId"))] =
+                    NColumnShard::TInternalPathId::FromRawValue(GetUint64(row.at("InternalPathId")));
+            }
+            UNIT_ASSERT_VALUES_EQUAL(result.size(), 1);
+            const auto& pathId = result.begin()->first;
+            UNIT_ASSERT_VALUES_EQUAL(result[pathId].size(), tableShardsCount);
+
+            for (const auto& [tabletId, pathIdTranslator]  : tablets) {
+                if (const auto& internalPathId = pathIdTranslator->ResolveInternalPathId(pathId)) {
+                    UNIT_ASSERT(result[pathId].contains(tabletId));
+                    UNIT_ASSERT_VALUES_EQUAL(result[pathId][tabletId], *internalPathId);
+                }
+            }
+        }
+    }
+
+    Y_UNIT_TEST(GranulePathId_Standalone) {
+        auto settings = TKikimrSettings().SetWithSampleTables(false);
+        TKikimrRunner kikimr(settings);
+
+        auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>();
+        auto helper = TLocalHelper(kikimr);
+        const ui32 tableShardsCount = 1201;
+        helper.CreateTestOlapStandaloneTable("table", tableShardsCount);
+        const auto tablets = csController->GetActiveTablets();
+        UNIT_ASSERT_VALUES_EQUAL(tablets.size(), tableShardsCount);
+        auto tableClient = kikimr.GetTableClient();
+        auto selectQuery = TString(R"(
+            SELECT PathId, TabletId, InternalPathId, 
+            FROM `/Root/table/.sys/primary_index_granule_stats`
+        )");
+        auto rows = ExecuteScanQuery(tableClient, selectQuery, true);
+        UNIT_ASSERT_VALUES_EQUAL(rows.size(), tableShardsCount);
+        THashMap<NColumnShard::TSchemeShardLocalPathId, THashMap<ui64, NColumnShard::TInternalPathId>> result;
+        for (const auto& row : rows) {
+            result[NColumnShard::TSchemeShardLocalPathId::FromRawValue(GetUint64(row.at("PathId")))][GetUint64(row.at("TabletId"))] =
+                NColumnShard::TInternalPathId::FromRawValue(GetUint64(row.at("InternalPathId")));
+        }
+        UNIT_ASSERT_VALUES_EQUAL(result.size(), 1);
+        const auto& pathId = result.begin()->first;
+        UNIT_ASSERT_VALUES_EQUAL(result[pathId].size(), tableShardsCount);
+
+        for (const auto& [tabletId, pathIdTranslator]  : tablets) {
+            const auto& internalPathId = pathIdTranslator->ResolveInternalPathId(pathId);
+            UNIT_ASSERT(internalPathId.has_value());
+            UNIT_ASSERT(result[pathId].contains(tabletId));
+            UNIT_ASSERT_VALUES_EQUAL(result[pathId][tabletId], *internalPathId);
+        }
+    }
+
     Y_UNIT_TEST(StatsSysView) {
         auto settings = TKikimrSettings()
             .SetWithSampleTables(false).SetColumnShardAlterObjectEnabled(true);
diff --git a/ydb/core/kqp/ut/olap/tiering_ut.cpp b/ydb/core/kqp/ut/olap/tiering_ut.cpp
index 401ee587435..44536d2e83a 100644
--- a/ydb/core/kqp/ut/olap/tiering_ut.cpp
+++ b/ydb/core/kqp/ut/olap/tiering_ut.cpp
@@ -164,7 +164,7 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) {
         auto& testHelper = tieringHelper.GetTestHelper();
         tieringHelper.SetTablePath("/Root/olapTable");
 
-        olapHelper.CreateTestOlapTableWithoutStore();
+        olapHelper.CreateTestOlapStandaloneTable();
         testHelper.CreateTier("tier1");
         testHelper.SetTiering("/Root/olapTable", DEFAULT_TIER_NAME, DEFAULT_COLUMN_NAME);
         {
diff --git a/ydb/core/sys_view/common/schema.h b/ydb/core/sys_view/common/schema.h
index 52780d1321b..d76aeffd6f3 100644
--- a/ydb/core/sys_view/common/schema.h
+++ b/ydb/core/sys_view/common/schema.h
@@ -604,6 +604,7 @@ struct Schema : NIceDb::Schema {
         struct PortionsCount: Column<3, NScheme::NTypeIds::Uint64> {};
         struct HostName: Column<4, NScheme::NTypeIds::Utf8> {};
         struct NodeId: Column<5, NScheme::NTypeIds::Uint64> {};
+        struct InternalPathId: Column<6, NScheme::NTypeIds::Uint64> {};
 
         using TKey = TableKey<PathId, TabletId>;
         using TColumns = TableColumns<
@@ -611,7 +612,8 @@ struct Schema : NIceDb::Schema {
             TabletId,
             PortionsCount,
             HostName,
-            NodeId
+            NodeId,
+            InternalPathId
         >;
     };
 
diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/granules/granules.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/granules/granules.cpp
index 0a58782671b..3cbbf9c7e59 100644
--- a/ydb/core/tx/columnshard/engines/reader/sys_view/granules/granules.cpp
+++ b/ydb/core/tx/columnshard/engines/reader/sys_view/granules/granules.cpp
@@ -14,6 +14,7 @@ bool TStatsIterator::AppendStats(const std::vector<std::unique_ptr<arrow::ArrayB
     NArrow::Append<arrow::UInt64Type>(*builders[2], granule.GetPortions().size());
     NArrow::Append<arrow::StringType>(*builders[3], HostNameField);
     NArrow::Append<arrow::UInt64Type>(*builders[4], NActors::TActivationContext::AsActorContext().SelfID.NodeId());
+    NArrow::Append<arrow::UInt64Type>(*builders[5], granule.GetPathId().InternalPathId.GetRawValue());
     return false;
 }