generate internal path id (#19660)

8 files changed, 53 insertions, 32 deletions

diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto
index f5fc5ee7430..3b9a3e288c7 100644
--- a/ydb/core/protos/config.proto
+++ b/ydb/core/protos/config.proto
@@ -1969,6 +1969,7 @@ message TColumnShardConfig {
     optional EJsonDoubleOutOfRangeHandlingPolicy DoubleOutOfRangeHandling = 40 [default = REJECT];
     optional bool PortionMetaV0Usage = 41 [default = true];
     optional uint32 PeriodicWakeupActivationPeriodMs = 42 [default = 60000];
+    optional bool GenerateInternalPathId = 43 [default = false]; //Be aware! IRREVERSIBLE(if set, can not be unset for a dababase)
 }
 
 message TSchemeShardConfig {
diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp
index 53a1372b4c5..7437b028f60 100644
--- a/ydb/core/tx/columnshard/columnshard_impl.cpp
+++ b/ydb/core/tx/columnshard/columnshard_impl.cpp
@@ -292,7 +292,7 @@ void TColumnShard::RunEnsureTable(const NKikimrTxColumnShard::TCreateTable& tabl
     }
     const auto internalPathId = TablesManager.CreateInternalPathId(schemeShardLocalPathId);
 
-    LOG_S_DEBUG("EnsureTable for pathId: " << TUnifiedPathId(internalPathId, schemeShardLocalPathId)
+    LOG_S_INFO("EnsureTable for pathId: " << TUnifiedPathId(internalPathId, schemeShardLocalPathId)
                                            << " ttl settings: " << tableProto.GetTtlSettings()
                                            << " at tablet " << TabletID());
 
diff --git a/ydb/core/tx/columnshard/columnshard_schema.h b/ydb/core/tx/columnshard/columnshard_schema.h
index d0551ddccf4..6be107778ad 100644
--- a/ydb/core/tx/columnshard/columnshard_schema.h
+++ b/ydb/core/tx/columnshard/columnshard_schema.h
@@ -83,7 +83,8 @@ struct Schema : NIceDb::Schema {
         GCBarrierPreparationGen = 16,
         GCBarrierPreparationStep = 17,
         SubDomainLocalPathId = 18,
-        SubDomainOutOfSpace = 19
+        SubDomainOutOfSpace = 19,
+        MaxInternalPathId = 20, //max internal path id ever known in this tablet
     };
 
     enum class EInsertTableIds : ui8 {
diff --git a/ydb/core/tx/columnshard/hooks/abstract/abstract.h b/ydb/core/tx/columnshard/hooks/abstract/abstract.h
index dbb538daff8..490382372c7 100644
--- a/ydb/core/tx/columnshard/hooks/abstract/abstract.h
+++ b/ydb/core/tx/columnshard/hooks/abstract/abstract.h
@@ -357,10 +357,6 @@ public:
         return {};
     }
 
-    virtual ui64 GetInternalPathIdOffset(const ui64 /*tabletId*/) const {
-        return 0;
-    }
-
     virtual void OnAddPathId(const ui64 /* tabletId */, const NColumnShard::TUnifiedPathId& /* pathId */) {
     }
     virtual void OnDeletePathId(const ui64 /* tabletId */, const NColumnShard::TUnifiedPathId& /* pathId */) {
diff --git a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h
index 851e6566b08..8646d349405 100644
--- a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h
+++ b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h
@@ -179,9 +179,6 @@ public:
         }
     }
 
-    virtual ui64 GetInternalPathIdOffset(const ui64 tabletId) const override {
-        return 1000000 + static_cast<ui64>(tabletId) % 1000;
-    }
 };
 
 }
diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp
index 08ba00ef3ee..c1f33ce06e7 100644
--- a/ydb/core/tx/columnshard/tables_manager.cpp
+++ b/ydb/core/tx/columnshard/tables_manager.cpp
@@ -65,6 +65,15 @@ bool TTablesManager::FillMonitoringReport(NTabletFlatExecutor::TTransactionConte
 
 bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) {
     {
+        std::optional<ui64> maxPathId;
+        if (!Schema::GetSpecialValueOpt(db, Schema::EValueIds::MaxInternalPathId, maxPathId)) {
+            return false;
+        }
+        if (maxPathId) {
+            MaxInternalPathId =TInternalPathId::FromRawValue(*maxPathId);
+        }
+    }
+    {
         TLoadTimeSignals::TLoadTimer timer = LoadTimeCounters->TableLoadTimeCounters.StartGuard();
         TMemoryProfileGuard g("TTablesManager/InitFromDB::Tables");
         auto rowset = db.Table<Schema::TableInfo>().Select();
@@ -79,6 +88,7 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) {
                 AFL_VERIFY(PathsToDrop[table.GetDropVersionVerified()].emplace(table.GetPathId().InternalPathId).second);
             }
             const auto& pathId = table.GetPathId();
+            AFL_VERIFY(pathId.InternalPathId <= MaxInternalPathId)("path_id", pathId)("max_internal_path_id", MaxInternalPathId);
             AFL_VERIFY(Tables.emplace(pathId.InternalPathId, std::move(table)).second);
             AFL_VERIFY(SchemeShardLocalToInternal.emplace(pathId.SchemeShardLocalPathId, pathId.InternalPathId).second);
 
@@ -226,9 +236,13 @@ bool TTablesManager::HasTable(const TInternalPathId pathId, const bool withDelet
 }
 
 TInternalPathId TTablesManager::CreateInternalPathId(const TSchemeShardLocalPathId schemeShardLocalPathId) {
-    const auto result = TInternalPathId::FromRawValue(schemeShardLocalPathId.GetRawValue() + InternalPathIdOffset);
-    AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("method", "CreateInternalPathId")("schemeShardLocalPathId", schemeShardLocalPathId)("result", result);
-    return result;
+    if (AppData()->ColumnShardConfig.GetGenerateInternalPathId()) {
+        const auto result = TInternalPathId::FromRawValue(MaxInternalPathId.GetRawValue() + 1);
+        MaxInternalPathId = result;
+        return result;
+    } else {
+        return TInternalPathId::FromRawValue(schemeShardLocalPathId.GetRawValue());
+    }
 }
 
 bool TTablesManager::IsReadyForStartWrite(const TInternalPathId pathId, const bool withDeleted) const {
@@ -281,6 +295,7 @@ void TTablesManager::RegisterTable(TTableInfo&& table, NIceDb::TNiceDb& db) {
     AFL_VERIFY(Tables.emplace(pathId, std::move(table)).second)("path_id", pathId)("size", Tables.size());
     AFL_VERIFY(SchemeShardLocalToInternal.emplace(table.GetPathId().SchemeShardLocalPathId, table.GetPathId().InternalPathId).second);
     Schema::SaveTableSchemeShardLocalPathId(db, table.GetPathId().InternalPathId, table.GetPathId().SchemeShardLocalPathId);
+    Schema::SaveSpecialValue(db, Schema::EValueIds::MaxInternalPathId, MaxInternalPathId.GetRawValue());
     if (PrimaryIndex) {
         PrimaryIndex->RegisterTable(pathId);
     }
@@ -369,14 +384,6 @@ void TTablesManager::AddTableVersion(const TInternalPathId pathId, const NOlap::
     table.AddVersion(version);
 }
 
-namespace {
-
-ui64 GetInternalPathIdOffset(ui64 tabletId) {
-    return NYDBTest::TControllers::GetColumnShardController()->GetInternalPathIdOffset(tabletId);
-}
-
-} //namespace
-
 TTablesManager::TTablesManager(const std::shared_ptr<NOlap::IStoragesManager>& storagesManager,
     const std::shared_ptr<NOlap::NDataAccessorControl::IDataAccessorsManager>& dataAccessorsManager,
     const std::shared_ptr<NOlap::TSchemaObjectsCache>& schemaCache, const std::shared_ptr<TPortionIndexStats>& portionsStats,
@@ -386,8 +393,7 @@ TTablesManager::TTablesManager(const std::shared_ptr<NOlap::IStoragesManager>& s
     , LoadTimeCounters(std::make_unique<TTableLoadTimeCounters>())
     , SchemaObjectsCache(schemaCache)
     , PortionsStats(portionsStats)
-    , TabletId(tabletId)
-    , InternalPathIdOffset(GetInternalPathIdOffset(tabletId)) {
+    , TabletId(tabletId) {
 }
 
 bool TTablesManager::TryFinalizeDropPathOnExecute(NTable::TDatabase& dbTable, const TInternalPathId pathId) const {
diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h
index 0d2ef20f865..79a9ced3945 100644
--- a/ydb/core/tx/columnshard/tables_manager.h
+++ b/ydb/core/tx/columnshard/tables_manager.h
@@ -210,7 +210,8 @@ private:
     NBackgroundTasks::TControlInterfaceContainer<NOlap::TSchemaObjectsCache> SchemaObjectsCache;
     std::shared_ptr<TPortionIndexStats> PortionsStats;
     ui64 TabletId = 0;
-    ui64 InternalPathIdOffset;
+    static constexpr ui64 MaxInternalPathIdDefault = 1'000'000'000; //Use a value presumably greater than any really used
+    TInternalPathId MaxInternalPathId = TInternalPathId::FromRawValue(MaxInternalPathIdDefault); //Max internal path id ever used in this tablet
 
     friend class TTxInit;
 
diff --git a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp
index f76305e1b4d..cc232fa7091 100644
--- a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp
+++ b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp
@@ -865,9 +865,10 @@ void TestExport(bool reboot, TExportTestOpts&& opts = TExportTestOpts{}) {
     }
 }
 
-void TestDrop(bool reboots) {
+void TestDrop(bool reboots, bool generateInternalPathId) {
     TTestBasicRuntime runtime;
     TTester::Setup(runtime);
+    runtime.GetAppData().ColumnShardConfig.SetGenerateInternalPathId(generateInternalPathId);
     auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard<TDefaultTestsController>();
 
     TActorId sender = runtime.AllocateEdgeActor();
@@ -1035,7 +1036,7 @@ extern bool gAllowLogBatchingDefaultValue;
 
 Y_UNIT_TEST_SUITE(TColumnShardTestSchema) {
 
-    Y_UNIT_TEST(CreateTable) {
+    void CreateTable(bool reboots, bool generateInternalPathId) {
         ui64 tableId = 1;
 
         std::vector<TTypeId> intTypes = {
@@ -1058,6 +1059,7 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) {
 
         TTestBasicRuntime runtime;
         TTester::Setup(runtime);
+        runtime.GetAppData().ColumnShardConfig.SetGenerateInternalPathId(generateInternalPathId);
         auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard<TDefaultTestsController>();
 
         using namespace NTxUT;
@@ -1084,6 +1086,24 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) {
             planStep = SetupSchema(runtime, sender, txBody, txId++);
         }
 
+        {
+            const auto& pathIdTranslator = csDefaultControllerGuard->GetPathIdTranslator(TTestTxConfig::TxTablet0);
+            const auto pathIds = pathIdTranslator->GetSchemeShardLocalPathIds();
+            for (const auto& pathId : pathIds) {
+                const auto& internalPathId = pathIdTranslator->ResolveInternalPathId(pathId);
+                UNIT_ASSERT(internalPathId);
+                if (generateInternalPathId) {
+                    UNIT_ASSERT_VALUES_UNEQUAL(internalPathId->GetRawValue(), pathId.GetRawValue());
+                } else {
+                    UNIT_ASSERT_VALUES_EQUAL(internalPathId->GetRawValue(), pathId.GetRawValue());
+                }
+            }
+        }
+
+        if (reboots) {
+            RebootTablet(runtime, TTestTxConfig::TxTablet0, sender);
+        }
+
         // TODO: support float types
         std::vector<TTypeId> floatTypes = {
             NTypeIds::Float,
@@ -1123,6 +1143,10 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) {
         }
     }
 
+    Y_UNIT_TEST_QUATRO(CreateTable, Reboots, GenerateInternalPathId) {
+        CreateTable(Reboots, GenerateInternalPathId);
+    }
+
     Y_UNIT_TEST_OCTO(TTL, Reboot, Internal, FirstPkColumn) {
         for (auto typeId : { NTypeIds::Timestamp, NTypeIds::Datetime, NTypeIds::Date, NTypeIds::Uint32, NTypeIds::Uint64 }) {
             Cerr << "Running TestTtl ttlColumnType=" << NKikimr::NScheme::TypeName(typeId) << Endl;
@@ -1236,14 +1260,9 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) {
         TestCompaction();
     }
 
-    Y_UNIT_TEST(Drop) {
-        TestDrop(false);
+    Y_UNIT_TEST_QUATRO(Drop, Reboots, GenerateInternalPathId) {
+        TestDrop(Reboots, GenerateInternalPathId);
     }
-
-    Y_UNIT_TEST(RebootDrop) {
-        TestDrop(true);
-    }
-
     Y_UNIT_TEST(DropWriteRace) {
         TestDropWriteRace();
     }