From 2c559a218839ade95e72cdc1e4dba7ccb208dede Mon Sep 17 00:00:00 2001 From: zverevgeny Date: Sat, 14 Jun 2025 20:03:55 +0300 Subject: generate internal path id (#19660) --- ydb/core/protos/config.proto | 1 + ydb/core/tx/columnshard/columnshard_impl.cpp | 2 +- ydb/core/tx/columnshard/columnshard_schema.h | 3 +- ydb/core/tx/columnshard/hooks/abstract/abstract.h | 4 --- .../tx/columnshard/hooks/testing/ro_controller.h | 3 -- ydb/core/tx/columnshard/tables_manager.cpp | 32 +++++++++++-------- ydb/core/tx/columnshard/tables_manager.h | 3 +- .../ut_schema/ut_columnshard_schema.cpp | 37 ++++++++++++++++------ 8 files changed, 53 insertions(+), 32 deletions(-) diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index f5fc5ee7430..3b9a3e288c7 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -1969,6 +1969,7 @@ message TColumnShardConfig { optional EJsonDoubleOutOfRangeHandlingPolicy DoubleOutOfRangeHandling = 40 [default = REJECT]; optional bool PortionMetaV0Usage = 41 [default = true]; optional uint32 PeriodicWakeupActivationPeriodMs = 42 [default = 60000]; + optional bool GenerateInternalPathId = 43 [default = false]; //Be aware! IRREVERSIBLE(if set, can not be unset for a dababase) } message TSchemeShardConfig { diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 53a1372b4c5..7437b028f60 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -292,7 +292,7 @@ void TColumnShard::RunEnsureTable(const NKikimrTxColumnShard::TCreateTable& tabl } const auto internalPathId = TablesManager.CreateInternalPathId(schemeShardLocalPathId); - LOG_S_DEBUG("EnsureTable for pathId: " << TUnifiedPathId(internalPathId, schemeShardLocalPathId) + LOG_S_INFO("EnsureTable for pathId: " << TUnifiedPathId(internalPathId, schemeShardLocalPathId) << " ttl settings: " << tableProto.GetTtlSettings() << " at tablet " << TabletID()); diff --git a/ydb/core/tx/columnshard/columnshard_schema.h b/ydb/core/tx/columnshard/columnshard_schema.h index d0551ddccf4..6be107778ad 100644 --- a/ydb/core/tx/columnshard/columnshard_schema.h +++ b/ydb/core/tx/columnshard/columnshard_schema.h @@ -83,7 +83,8 @@ struct Schema : NIceDb::Schema { GCBarrierPreparationGen = 16, GCBarrierPreparationStep = 17, SubDomainLocalPathId = 18, - SubDomainOutOfSpace = 19 + SubDomainOutOfSpace = 19, + MaxInternalPathId = 20, //max internal path id ever known in this tablet }; enum class EInsertTableIds : ui8 { diff --git a/ydb/core/tx/columnshard/hooks/abstract/abstract.h b/ydb/core/tx/columnshard/hooks/abstract/abstract.h index dbb538daff8..490382372c7 100644 --- a/ydb/core/tx/columnshard/hooks/abstract/abstract.h +++ b/ydb/core/tx/columnshard/hooks/abstract/abstract.h @@ -357,10 +357,6 @@ public: return {}; } - virtual ui64 GetInternalPathIdOffset(const ui64 /*tabletId*/) const { - return 0; - } - virtual void OnAddPathId(const ui64 /* tabletId */, const NColumnShard::TUnifiedPathId& /* pathId */) { } virtual void OnDeletePathId(const ui64 /* tabletId */, const NColumnShard::TUnifiedPathId& /* pathId */) { diff --git a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h index 851e6566b08..8646d349405 100644 --- a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h +++ b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h @@ -179,9 +179,6 @@ public: } } - virtual ui64 GetInternalPathIdOffset(const ui64 tabletId) const override { - return 1000000 + static_cast(tabletId) % 1000; - } }; } diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index 08ba00ef3ee..c1f33ce06e7 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -64,6 +64,15 @@ bool TTablesManager::FillMonitoringReport(NTabletFlatExecutor::TTransactionConte } bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { + { + std::optional maxPathId; + if (!Schema::GetSpecialValueOpt(db, Schema::EValueIds::MaxInternalPathId, maxPathId)) { + return false; + } + if (maxPathId) { + MaxInternalPathId =TInternalPathId::FromRawValue(*maxPathId); + } + } { TLoadTimeSignals::TLoadTimer timer = LoadTimeCounters->TableLoadTimeCounters.StartGuard(); TMemoryProfileGuard g("TTablesManager/InitFromDB::Tables"); @@ -79,6 +88,7 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { AFL_VERIFY(PathsToDrop[table.GetDropVersionVerified()].emplace(table.GetPathId().InternalPathId).second); } const auto& pathId = table.GetPathId(); + AFL_VERIFY(pathId.InternalPathId <= MaxInternalPathId)("path_id", pathId)("max_internal_path_id", MaxInternalPathId); AFL_VERIFY(Tables.emplace(pathId.InternalPathId, std::move(table)).second); AFL_VERIFY(SchemeShardLocalToInternal.emplace(pathId.SchemeShardLocalPathId, pathId.InternalPathId).second); @@ -226,9 +236,13 @@ bool TTablesManager::HasTable(const TInternalPathId pathId, const bool withDelet } TInternalPathId TTablesManager::CreateInternalPathId(const TSchemeShardLocalPathId schemeShardLocalPathId) { - const auto result = TInternalPathId::FromRawValue(schemeShardLocalPathId.GetRawValue() + InternalPathIdOffset); - AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("method", "CreateInternalPathId")("schemeShardLocalPathId", schemeShardLocalPathId)("result", result); - return result; + if (AppData()->ColumnShardConfig.GetGenerateInternalPathId()) { + const auto result = TInternalPathId::FromRawValue(MaxInternalPathId.GetRawValue() + 1); + MaxInternalPathId = result; + return result; + } else { + return TInternalPathId::FromRawValue(schemeShardLocalPathId.GetRawValue()); + } } bool TTablesManager::IsReadyForStartWrite(const TInternalPathId pathId, const bool withDeleted) const { @@ -281,6 +295,7 @@ void TTablesManager::RegisterTable(TTableInfo&& table, NIceDb::TNiceDb& db) { AFL_VERIFY(Tables.emplace(pathId, std::move(table)).second)("path_id", pathId)("size", Tables.size()); AFL_VERIFY(SchemeShardLocalToInternal.emplace(table.GetPathId().SchemeShardLocalPathId, table.GetPathId().InternalPathId).second); Schema::SaveTableSchemeShardLocalPathId(db, table.GetPathId().InternalPathId, table.GetPathId().SchemeShardLocalPathId); + Schema::SaveSpecialValue(db, Schema::EValueIds::MaxInternalPathId, MaxInternalPathId.GetRawValue()); if (PrimaryIndex) { PrimaryIndex->RegisterTable(pathId); } @@ -369,14 +384,6 @@ void TTablesManager::AddTableVersion(const TInternalPathId pathId, const NOlap:: table.AddVersion(version); } -namespace { - -ui64 GetInternalPathIdOffset(ui64 tabletId) { - return NYDBTest::TControllers::GetColumnShardController()->GetInternalPathIdOffset(tabletId); -} - -} //namespace - TTablesManager::TTablesManager(const std::shared_ptr& storagesManager, const std::shared_ptr& dataAccessorsManager, const std::shared_ptr& schemaCache, const std::shared_ptr& portionsStats, @@ -386,8 +393,7 @@ TTablesManager::TTablesManager(const std::shared_ptr& s , LoadTimeCounters(std::make_unique()) , SchemaObjectsCache(schemaCache) , PortionsStats(portionsStats) - , TabletId(tabletId) - , InternalPathIdOffset(GetInternalPathIdOffset(tabletId)) { + , TabletId(tabletId) { } bool TTablesManager::TryFinalizeDropPathOnExecute(NTable::TDatabase& dbTable, const TInternalPathId pathId) const { diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h index 0d2ef20f865..79a9ced3945 100644 --- a/ydb/core/tx/columnshard/tables_manager.h +++ b/ydb/core/tx/columnshard/tables_manager.h @@ -210,7 +210,8 @@ private: NBackgroundTasks::TControlInterfaceContainer SchemaObjectsCache; std::shared_ptr PortionsStats; ui64 TabletId = 0; - ui64 InternalPathIdOffset; + static constexpr ui64 MaxInternalPathIdDefault = 1'000'000'000; //Use a value presumably greater than any really used + TInternalPathId MaxInternalPathId = TInternalPathId::FromRawValue(MaxInternalPathIdDefault); //Max internal path id ever used in this tablet friend class TTxInit; diff --git a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp index f76305e1b4d..cc232fa7091 100644 --- a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp +++ b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp @@ -865,9 +865,10 @@ void TestExport(bool reboot, TExportTestOpts&& opts = TExportTestOpts{}) { } } -void TestDrop(bool reboots) { +void TestDrop(bool reboots, bool generateInternalPathId) { TTestBasicRuntime runtime; TTester::Setup(runtime); + runtime.GetAppData().ColumnShardConfig.SetGenerateInternalPathId(generateInternalPathId); auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); TActorId sender = runtime.AllocateEdgeActor(); @@ -1035,7 +1036,7 @@ extern bool gAllowLogBatchingDefaultValue; Y_UNIT_TEST_SUITE(TColumnShardTestSchema) { - Y_UNIT_TEST(CreateTable) { + void CreateTable(bool reboots, bool generateInternalPathId) { ui64 tableId = 1; std::vector intTypes = { @@ -1058,6 +1059,7 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) { TTestBasicRuntime runtime; TTester::Setup(runtime); + runtime.GetAppData().ColumnShardConfig.SetGenerateInternalPathId(generateInternalPathId); auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); using namespace NTxUT; @@ -1084,6 +1086,24 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) { planStep = SetupSchema(runtime, sender, txBody, txId++); } + { + const auto& pathIdTranslator = csDefaultControllerGuard->GetPathIdTranslator(TTestTxConfig::TxTablet0); + const auto pathIds = pathIdTranslator->GetSchemeShardLocalPathIds(); + for (const auto& pathId : pathIds) { + const auto& internalPathId = pathIdTranslator->ResolveInternalPathId(pathId); + UNIT_ASSERT(internalPathId); + if (generateInternalPathId) { + UNIT_ASSERT_VALUES_UNEQUAL(internalPathId->GetRawValue(), pathId.GetRawValue()); + } else { + UNIT_ASSERT_VALUES_EQUAL(internalPathId->GetRawValue(), pathId.GetRawValue()); + } + } + } + + if (reboots) { + RebootTablet(runtime, TTestTxConfig::TxTablet0, sender); + } + // TODO: support float types std::vector floatTypes = { NTypeIds::Float, @@ -1123,6 +1143,10 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) { } } + Y_UNIT_TEST_QUATRO(CreateTable, Reboots, GenerateInternalPathId) { + CreateTable(Reboots, GenerateInternalPathId); + } + Y_UNIT_TEST_OCTO(TTL, Reboot, Internal, FirstPkColumn) { for (auto typeId : { NTypeIds::Timestamp, NTypeIds::Datetime, NTypeIds::Date, NTypeIds::Uint32, NTypeIds::Uint64 }) { Cerr << "Running TestTtl ttlColumnType=" << NKikimr::NScheme::TypeName(typeId) << Endl; @@ -1236,14 +1260,9 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) { TestCompaction(); } - Y_UNIT_TEST(Drop) { - TestDrop(false); + Y_UNIT_TEST_QUATRO(Drop, Reboots, GenerateInternalPathId) { + TestDrop(Reboots, GenerateInternalPathId); } - - Y_UNIT_TEST(RebootDrop) { - TestDrop(true); - } - Y_UNIT_TEST(DropWriteRace) { TestDropWriteRace(); } -- cgit v1.3