diff options
author | ivanmorozov <ivanmorozov@yandex-team.com> | 2022-11-24 19:07:11 +0300 |
---|---|---|
committer | ivanmorozov <ivanmorozov@yandex-team.com> | 2022-11-24 19:07:11 +0300 |
commit | 6bb204fd7976045e3c045f7fc7cfdbb836e86d46 (patch) | |
tree | 89fc57e7303a82e30fa4f73a5b1fa55639b9e5f0 | |
parent | 9d83e592bfb14abd1e752700722aaf834685206d (diff) | |
download | ydb-6bb204fd7976045e3c045f7fc7cfdbb836e86d46.tar.gz |
correct tiering model (composite tiering rule + direct tiering link in ss)
49 files changed, 487 insertions, 616 deletions
diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h index 7cbd1868460..83da6779e76 100644 --- a/ydb/core/base/events.h +++ b/ydb/core/base/events.h @@ -155,7 +155,7 @@ struct TKikimrEvents : TEvents { ES_METADATA_INITIALIZER, ES_YDB_AUDIT_LOG, ES_METADATA_MANAGER, - ES_METADATA_SECRET + ES_METADATA_SECRET, }; }; diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 1cc87172440..9623776b7b1 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -471,7 +471,7 @@ message TColumnDataLifeCycle { } message TStorageTiering { - optional bool EnableTiering = 1[default = false]; + optional string UseTiering = 1; } message TDisabled { @@ -880,7 +880,7 @@ message TS3Settings { message TTaskCleaner { optional uint64 PathId = 1; optional TS3Settings StorageSettings = 2; - optional string OwnerPath = 3; + optional string TieringId = 3; } message TBackupTask { diff --git a/ydb/core/testlib/common_helper.cpp b/ydb/core/testlib/common_helper.cpp index eb9a9344447..f8bf0d87ec7 100644 --- a/ydb/core/testlib/common_helper.cpp +++ b/ydb/core/testlib/common_helper.cpp @@ -37,7 +37,8 @@ void THelper::StartDataRequest(const TString& request, const bool expectSuccess) } void THelper::StartSchemaRequest(const TString& request, const bool expectSuccess) const { - NYdb::NTable::TTableClient tClient(Server.GetDriver(), NYdb::NTable::TClientSettings().UseQueryCache(false)); + NYdb::NTable::TTableClient tClient(Server.GetDriver(), + NYdb::NTable::TClientSettings().UseQueryCache(false).AuthToken("root@builtin")); auto expectation = expectSuccess; tClient.CreateSession().Subscribe([request, expectation](NThreading::TFuture<NYdb::NTable::TCreateSessionResult> f) { auto session = f.GetValueSync().GetSession(); diff --git a/ydb/core/tx/columnshard/columnshard.cpp b/ydb/core/tx/columnshard/columnshard.cpp index 460e07b02b6..1c2807c66fb 100644 --- a/ydb/core/tx/columnshard/columnshard.cpp +++ b/ydb/core/tx/columnshard/columnshard.cpp @@ -30,7 +30,7 @@ void TColumnShard::SwitchToWork(const TActorContext& ctx) { CompactionActor = ctx.Register(CreateCompactionActor(TabletID(), ctx.SelfID)); EvictionActor = ctx.Register(CreateEvictionActor(TabletID(), ctx.SelfID)); for (auto&& i : Tables) { - ActivateTiering(i.first, i.second.TieringEnabled); + ActivateTiering(i.first, i.second.TieringUsage); } SignalTabletActive(ctx); } diff --git a/ydb/core/tx/columnshard/columnshard__init.cpp b/ydb/core/tx/columnshard/columnshard__init.cpp index d1351ca2487..15ee07d9e55 100644 --- a/ydb/core/tx/columnshard/columnshard__init.cpp +++ b/ydb/core/tx/columnshard/columnshard__init.cpp @@ -204,7 +204,7 @@ bool TTxInit::ReadEverything(TTransactionContext& txc, const TActorContext& ctx) const ui64 pathId = rowset.GetValue<Schema::TableInfo::PathId>(); auto& table = Self->Tables[pathId]; table.PathId = pathId; - table.TieringEnabled = rowset.GetValue<Schema::TableInfo::TieringEnabled>() == "1"; + table.TieringUsage = rowset.GetValue<Schema::TableInfo::TieringUsage>(); if (rowset.HaveValue<Schema::TableInfo::DropStep>() && rowset.HaveValue<Schema::TableInfo::DropTxId>()) { diff --git a/ydb/core/tx/columnshard/columnshard__write_index.cpp b/ydb/core/tx/columnshard/columnshard__write_index.cpp index 22ec60a381e..e6c88ef4cd9 100644 --- a/ydb/core/tx/columnshard/columnshard__write_index.cpp +++ b/ydb/core/tx/columnshard/columnshard__write_index.cpp @@ -137,7 +137,7 @@ bool TTxWriteIndex::Execute(TTransactionContext& txc, const TActorContext& ctx) } // Mark exported blobs - auto& tManager = Self->GetTierManagerVerified(NTiers::TGlobalTierId(Self->OwnerPath, tierName)); + auto& tManager = Self->GetTierManagerVerified(tierName); if (tManager.NeedExport()) { for (auto& rec : portionInfo.Records) { auto& blobId = rec.BlobRange.BlobId; diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 3aa13a26915..38cdf1da14a 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -509,8 +509,8 @@ void TColumnShard::RunEnsureTable(const NKikimrTxColumnShard::TCreateTable& tabl Ttl.SetPathTtl(pathId, TTtl::TDescription(ttlInfo)); SetCounter(COUNTER_TABLE_TTLS, Ttl.PathsCount()); } else if (ttlInfo.HasTiering()) { - table.TieringEnabled = ttlInfo.GetTiering().GetEnableTiering(); - ActivateTiering(pathId, table.TieringEnabled); + table.TieringUsage = ttlInfo.GetTiering().GetUseTiering(); + ActivateTiering(pathId, table.TieringUsage); } } @@ -527,7 +527,7 @@ void TColumnShard::RunEnsureTable(const NKikimrTxColumnShard::TCreateTable& tabl tableVerProto.SetSchemaPresetVersionAdj(tableProto.GetSchemaPresetVersionAdj()); tableVerProto.SetTtlSettingsPresetVersionAdj(tableProto.GetTtlSettingsPresetVersionAdj()); - Schema::SaveTableInfo(db, table.PathId, table.TieringEnabled); + Schema::SaveTableInfo(db, table.PathId, table.TieringUsage); Schema::SaveTableVersionInfo(db, table.PathId, version, tableVerProto); SetCounter(COUNTER_TABLES, Tables.size()); } else { @@ -553,8 +553,8 @@ void TColumnShard::RunAlterTable(const NKikimrTxColumnShard::TAlterTable& alterP info.SetSchemaPresetId(EnsureSchemaPreset(db, alterProto.GetSchemaPreset(), version)); } - const bool tieringEnabled = alterProto.HasTtlSettings() && alterProto.GetTtlSettings().HasTiering() && alterProto.GetTtlSettings().GetTiering().GetEnableTiering(); - ActivateTiering(pathId, tieringEnabled); + const TString& tieringUsage = alterProto.GetTtlSettings().GetTiering().GetUseTiering(); + ActivateTiering(pathId, tieringUsage); if (alterProto.HasTtlSettings() && alterProto.GetTtlSettings().HasEnabled()) { *info.MutableTtlSettings() = alterProto.GetTtlSettings(); Ttl.SetPathTtl(pathId, TTtl::TDescription(alterProto.GetTtlSettings())); @@ -563,7 +563,7 @@ void TColumnShard::RunAlterTable(const NKikimrTxColumnShard::TAlterTable& alterP } info.SetSchemaPresetVersionAdj(alterProto.GetSchemaPresetVersionAdj()); - Schema::SaveTableInfo(db, table.PathId, tieringEnabled); + Schema::SaveTableInfo(db, table.PathId, tieringUsage); Schema::SaveTableVersionInfo(db, table.PathId, version, info); } @@ -993,14 +993,14 @@ void TColumnShard::MapExternBlobs(const TActorContext& /*ctx*/, NOlap::TReadMeta void TColumnShard::ExportBlobs(const TActorContext& ctx, ui64 exportNo, const TString& tierName, TEvPrivate::TEvExport::TBlobDataMap&& blobsInfo) const { - if (auto s3 = GetS3ActorForTier(NTiers::TGlobalTierId(OwnerPath, tierName))) { + if (auto s3 = GetS3ActorForTier(tierName)) { auto event = std::make_unique<TEvPrivate::TEvExport>(exportNo, tierName, s3, std::move(blobsInfo)); ctx.Register(CreateExportActor(TabletID(), ctx.SelfID, event.release())); } } void TColumnShard::ForgetBlobs(const TActorContext& ctx, const TString& tierName, std::vector<NOlap::TEvictedBlob>&& blobs) const { - if (auto s3 = GetS3ActorForTier(NTiers::TGlobalTierId(OwnerPath, tierName))) { + if (auto s3 = GetS3ActorForTier(tierName)) { auto forget = std::make_unique<TEvPrivate::TEvForget>(); forget->Evicted = std::move(blobs); ctx.Send(s3, forget.release()); @@ -1009,7 +1009,7 @@ void TColumnShard::ForgetBlobs(const TActorContext& ctx, const TString& tierName bool TColumnShard::GetExportedBlob(const TActorContext& ctx, TActorId dst, ui64 cookie, const TString& tierName, NOlap::TEvictedBlob&& evicted, std::vector<NOlap::TBlobRange>&& ranges) { - if (auto s3 = GetS3ActorForTier(NTiers::TGlobalTierId(OwnerPath, tierName))) { + if (auto s3 = GetS3ActorForTier(tierName)) { auto get = std::make_unique<TEvPrivate::TEvGetExported>(); get->DstActor = dst; get->DstCookie = cookie; @@ -1031,7 +1031,7 @@ void TColumnShard::Die(const TActorContext& ctx) { return IActor::Die(ctx); } -TActorId TColumnShard::GetS3ActorForTier(const NTiers::TGlobalTierId& tierId) const { +TActorId TColumnShard::GetS3ActorForTier(const TString& tierId) const { if (!Tiers) { return {}; } @@ -1040,7 +1040,7 @@ TActorId TColumnShard::GetS3ActorForTier(const NTiers::TGlobalTierId& tierId) co void TColumnShard::Handle(NMetadataProvider::TEvRefreshSubscriberData::TPtr& ev) { Y_VERIFY(Tiers); - Tiers->TakeConfigs(ev->Get()->GetSnapshot()); + Tiers->TakeConfigs(ev->Get()->GetSnapshot(), nullptr); } NOlap::TIndexInfo TColumnShard::GetActualIndexInfo(const bool tiersUsage) const { @@ -1053,14 +1053,14 @@ NOlap::TIndexInfo TColumnShard::GetActualIndexInfo(const bool tiersUsage) const return indexInfo; } -void TColumnShard::ActivateTiering(const ui64 pathId, const bool enableTiering) { - if (OwnerPath && !Tiers) { +void TColumnShard::ActivateTiering(const ui64 pathId, const TString& useTiering) { + if (!Tiers) { Tiers = std::make_shared<TTiersManager>(TabletID(), SelfId()); Tiers->Start(Tiers); } if (!!Tiers) { - if (enableTiering) { - Tiers->EnablePathId(pathId); + if (useTiering) { + Tiers->EnablePathId(pathId, useTiering); } else { Tiers->DisablePathId(pathId); } diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 52057f905f3..dccaba39515 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -139,7 +139,7 @@ class TColumnShard Y_UNUSED(ctx); } - const NTiers::TManager& GetTierManagerVerified(const NTiers::TGlobalTierId& tierId) const { + const NTiers::TManager& GetTierManagerVerified(const TString& tierId) const { Y_VERIFY(!!Tiers); return Tiers->GetManagerVerified(tierId); } @@ -177,7 +177,7 @@ class TColumnShard NOlap::TIndexInfo GetActualIndexInfo(const bool tiersUsage = true) const; - void ActivateTiering(const ui64 pathId, const bool enableTiering); + void ActivateTiering(const ui64 pathId, const TString& useTiering); protected: STFUNC(StateInit) { TRACE_EVENT(NKikimrServices::TX_COLUMNSHARD); @@ -312,7 +312,7 @@ private: ui64 PathId; std::map<TRowVersion, TTableVersionInfo> Versions; TRowVersion DropVersion = TRowVersion::Max(); - bool TieringEnabled = false; + TString TieringUsage; bool IsDropped() const { return DropVersion != TRowVersion::Max(); @@ -449,7 +449,7 @@ private: NOlap::TIndexInfo ConvertSchema(const NKikimrSchemeOp::TColumnTableSchema& schema); void MapExternBlobs(const TActorContext& ctx, NOlap::TReadMetadata& metadata); - TActorId GetS3ActorForTier(const NTiers::TGlobalTierId& tierId) const; + TActorId GetS3ActorForTier(const TString& tierId) const; void ExportBlobs(const TActorContext& ctx, ui64 exportNo, const TString& tierName, TEvPrivate::TEvExport::TBlobDataMap&& blobsInfo) const; void ForgetBlobs(const TActorContext& ctx, const TString& tierName, std::vector<NOlap::TEvictedBlob>&& blobs) const; diff --git a/ydb/core/tx/columnshard/columnshard_schema.h b/ydb/core/tx/columnshard/columnshard_schema.h index 4a7ee6c8268..cf5069a7601 100644 --- a/ydb/core/tx/columnshard/columnshard_schema.h +++ b/ydb/core/tx/columnshard/columnshard_schema.h @@ -125,10 +125,10 @@ struct Schema : NIceDb::Schema { struct PathId : Column<1, NScheme::NTypeIds::Uint64> {}; struct DropStep : Column<2, NScheme::NTypeIds::Uint64> {}; struct DropTxId : Column<3, NScheme::NTypeIds::Uint64> {}; - struct TieringEnabled: Column<4, NScheme::NTypeIds::String> {}; + struct TieringUsage: Column<4, NScheme::NTypeIds::String> {}; using TKey = TableKey<PathId>; - using TColumns = TableColumns<PathId, DropStep, DropTxId, TieringEnabled>; + using TColumns = TableColumns<PathId, DropStep, DropTxId, TieringUsage>; }; struct TableVersionInfo : Table<11> { @@ -364,9 +364,9 @@ struct Schema : NIceDb::Schema { db.Table<SchemaPresetInfo>().Key(id).Delete(); } - static void SaveTableInfo(NIceDb::TNiceDb& db, const ui64 pathId, const bool tieringEnabled) { + static void SaveTableInfo(NIceDb::TNiceDb& db, const ui64 pathId, const TString tieringUsage) { db.Table<TableInfo>().Key(pathId).Update( - NIceDb::TUpdate<TableInfo::TieringEnabled>(tieringEnabled ? "1" : "0") + NIceDb::TUpdate<TableInfo::TieringUsage>(tieringUsage) ); } diff --git a/ydb/core/tx/columnshard/columnshard_ut_common.cpp b/ydb/core/tx/columnshard/columnshard_ut_common.cpp index 18f08925e92..2b954e8f32e 100644 --- a/ydb/core/tx/columnshard/columnshard_ut_common.cpp +++ b/ydb/core/tx/columnshard/columnshard_ut_common.cpp @@ -291,12 +291,20 @@ TSerializedTableRange MakeTestRange(std::pair<ui64, ui64> range, bool inclusiveF TConstArrayRef<TCell>(cellsTo), inclusiveTo); } -NMetadataProvider::ISnapshot::TPtr TTestSchema::BuildSnapshot(const TTableSpecials& specials, const TString& tablePath, const ui32 tablePathId) { +NMetadataProvider::ISnapshot::TPtr TTestSchema::BuildSnapshot(const TTableSpecials& specials) { std::unique_ptr<NColumnShard::NTiers::TConfigsSnapshot> cs(new NColumnShard::NTiers::TConfigsSnapshot(Now())); + if (specials.Tiers.empty()) { + return cs; + } + NColumnShard::NTiers::TTieringRule tRule; + tRule.SetTieringRuleId("Tiering1"); for (auto&& tier : specials.Tiers) { + if (!tRule.GetDefaultColumn()) { + tRule.SetDefaultColumn(tier.GetTtlColumn()); + } + Y_VERIFY(tRule.GetDefaultColumn() == tier.GetTtlColumn()); { NColumnShard::NTiers::TTierConfig tConfig; - tConfig.SetOwnerPath("/Root/olapStore"); tConfig.SetTierName(tier.Name); tConfig.MutableProtoConfig().SetName(tier.Name); auto& cProto = tConfig.MutableProtoConfig(); @@ -309,18 +317,11 @@ NMetadataProvider::ISnapshot::TPtr TTestSchema::BuildSnapshot(const TTableSpecia if (tier.CompressionLevel) { cProto.MutableCompression()->SetCompressionLevel(*tier.CompressionLevel); } - cs->MutableTierConfigs().emplace(tConfig.GetGlobalTierId(), tConfig); - } - { - NColumnShard::NTiers::TTieringRule tRule; - tRule.SetOwnerPath("/Root/olapStore"); - tRule.SetDurationForEvict(TDuration::Seconds(tier.GetEvictAfterSecondsUnsafe())); - tRule.SetTablePath(tablePath).SetTablePathId(tablePathId); - tRule.SetTierName(tier.Name); - tRule.SetColumn(tier.GetTtlColumn()); - cs->MutableTableTierings()[tablePath].AddRule(std::move(tRule)); + cs->MutableTierConfigs().emplace(tConfig.GetTierName(), tConfig); } + tRule.AddInterval(tier.Name, TDuration::Seconds(tier.GetEvictAfterSecondsUnsafe())); } + cs->MutableTableTierings().emplace(tRule.GetTieringRuleId(), tRule); return cs; } diff --git a/ydb/core/tx/columnshard/columnshard_ut_common.h b/ydb/core/tx/columnshard/columnshard_ut_common.h index a850e153f9d..8c4c76b9cb2 100644 --- a/ydb/core/tx/columnshard/columnshard_ut_common.h +++ b/ydb/core/tx/columnshard/columnshard_ut_common.h @@ -206,7 +206,7 @@ struct TTestSchema { static void InitTiers(const TTableSpecials& specials, NKikimrSchemeOp::TColumnDataLifeCycle* ttlSettings) { Y_VERIFY(specials.HasTiers()); - ttlSettings->MutableTiering()->SetEnableTiering(true); + ttlSettings->MutableTiering()->SetUseTiering("Tiering1"); } static TString CreateTableTxBody(ui64 pathId, const TVector<std::pair<TString, TTypeInfo>>& columns, @@ -288,7 +288,7 @@ struct TTestSchema { enable->SetColumnName(specials.GetTtlColumn()); enable->SetExpireAfterSeconds(specials.GetEvictAfterSecondsUnsafe()); } else if (specials.HasTiers()) { - ttlSettings->MutableTiering()->SetEnableTiering(true); + ttlSettings->MutableTiering()->SetUseTiering("Tiering1"); } else { ttlSettings->MutableDisabled(); } @@ -308,7 +308,7 @@ struct TTestSchema { return out; } - static NMetadataProvider::ISnapshot::TPtr BuildSnapshot(const TTableSpecials& specials, const TString& tablePath, const ui32 tablePathId); + static NMetadataProvider::ISnapshot::TPtr BuildSnapshot(const TTableSpecials& specials); static TString CommitTxBody(ui64 metaShard, const TVector<ui64>& writeIds) { NKikimrTxColumnShard::TCommitTxBody proto; diff --git a/ydb/core/tx/columnshard/ut_columnshard_schema.cpp b/ydb/core/tx/columnshard/ut_columnshard_schema.cpp index 21b883ae4c4..177d543072e 100644 --- a/ydb/core/tx/columnshard/ut_columnshard_schema.cpp +++ b/ydb/core/tx/columnshard/ut_columnshard_schema.cpp @@ -172,7 +172,7 @@ void TestTtl(bool reboots, bool internal, TTestSchema::TTableSpecials spec = {}, UNIT_ASSERT(ok); PlanSchemaTx(runtime, sender, {planStep, txId}); if (spec.HasTiers()) { - ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(spec, "test", tableId)); + ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(spec)); } // @@ -237,7 +237,7 @@ void TestTtl(bool reboots, bool internal, TTestSchema::TTableSpecials spec = {}, UNIT_ASSERT(ok); PlanSchemaTx(runtime, sender, {planStep, txId}); if (spec.HasTiers()) { - ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(spec, "test", tableId)); + ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(spec)); } if (internal) { @@ -270,7 +270,7 @@ void TestTtl(bool reboots, bool internal, TTestSchema::TTableSpecials spec = {}, {++planStep, ++txId}); UNIT_ASSERT(ok); if (spec.HasTiers()) { - ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(TTestSchema::TTableSpecials(), "test", tableId)); + ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(TTestSchema::TTableSpecials())); } PlanSchemaTx(runtime, sender, {planStep, txId}); @@ -445,7 +445,7 @@ TestTiers(bool reboots, const std::vector<TString>& blobs, const std::vector<TTe } PlanSchemaTx(runtime, sender, {planStep, txId}); if (specs[0].Tiers.size()) { - ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(specs[0], "test", tableId)); + ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(specs[0])); } for (auto& data : blobs) { @@ -484,7 +484,7 @@ TestTiers(bool reboots, const std::vector<TString>& blobs, const std::vector<TTe } } if (specs[i].Tiers.size()) { - ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(specs[i], "test", tableId)); + ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(specs[i])); } counter.SetRestartTabletOnPutData(reboots ? 1 : 0); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h b/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h index 8b62fff951a..618a5a6d1cd 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h @@ -188,7 +188,7 @@ public: Bill(operationId, txState.TargetPathId, shardIdx, ru, context); } - // TODO(ilnaz): backward compatability, remove it + // TODO(ilnaz): backward compatibility, remove it if (result.GetSuccess()) { if (result.HasBytesProcessed()) { txState.DataTotalSize += result.GetBytesProcessed(); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_olap_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_olap_table.cpp index 8c83c5470ed..01a2f90ef57 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_olap_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_olap_table.cpp @@ -309,20 +309,25 @@ public: Y_VERIFY(txState->TxType == TTxState::TxDropColumnTable); LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - DebugHint() << " ProgressState" - << ", at schemeshard: " << ssId); + DebugHint() << " ProgressState" + << ", at schemeshard: " << ssId); - if (NBackgroundTasks::TServiceOperator::IsEnabled()) { - NSchemeShard::TPath path = NSchemeShard::TPath::Init(txState->TargetPathId, context.SS); - TColumnTableInfo::TPtr tableInfo = context.SS->ColumnTables.at(path.Base()->PathId); - NSchemeShard::TPath ownerPath = tableInfo->IsStandalone() ? path : path.FindOlapStore(); - Y_VERIFY(path.IsResolved()); - NBackgroundTasks::TTask task(std::make_shared<NColumnShard::NTiers::TTaskCleanerActivity>(txState->TargetPathId.LocalPathId, ownerPath.PathString()), nullptr); + if (!NBackgroundTasks::TServiceOperator::IsEnabled()) { + return Finish(context); + } + NSchemeShard::TPath path = NSchemeShard::TPath::Init(txState->TargetPathId, context.SS); + TColumnTableInfo::TPtr tableInfo = context.SS->ColumnTables.at(path.Base()->PathId); + const TString& tieringId = tableInfo->Description.GetTtlSettings().GetTiering().GetUseTiering(); + if (!tieringId) { + return Finish(context); + } + + { + NBackgroundTasks::TTask task(std::make_shared<NColumnShard::NTiers::TTaskCleanerActivity>( + tieringId, txState->TargetPathId.LocalPathId), nullptr); task.SetId(OperationId.SerializeToString()); context.SS->SelfId().Send(NBackgroundTasks::MakeServiceId(context.SS->SelfId().NodeId()), new NBackgroundTasks::TEvAddTask(std::move(task))); return false; - } else { - return Finish(context); } } }; diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp index 7d73cfab229..adbb76e37ab 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp @@ -924,13 +924,13 @@ TCheckFunc HasColumnTableTtlSettingsDisabled() { }; } -TCheckFunc HasColumnTableTtlSettingsTiering(const TString& /*tierName*/) { +TCheckFunc HasColumnTableTtlSettingsTiering(const TString& tieringName) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { const auto& table = record.GetPathDescription().GetColumnTableDescription(); UNIT_ASSERT(table.HasTtlSettings()); const auto& ttl = table.GetTtlSettings(); UNIT_ASSERT(ttl.HasTiering()); - UNIT_ASSERT(ttl.GetTiering().GetEnableTiering()); + UNIT_ASSERT_EQUAL(ttl.GetTiering().GetUseTiering(), tieringName); }; } diff --git a/ydb/core/tx/schemeshard/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap.cpp index 6b98e69e8b4..f2a37356b0d 100644 --- a/ydb/core/tx/schemeshard/ut_olap.cpp +++ b/ydb/core/tx/schemeshard/ut_olap.cpp @@ -512,7 +512,7 @@ Y_UNIT_TEST_SUITE(TOlap) { Name: "Table3" TtlSettings { Tiering { - EnableTiering : true + UseTiering : "Tiering1" } } )"; @@ -524,13 +524,13 @@ Y_UNIT_TEST_SUITE(TOlap) { NLs::HasColumnTableSchemaPreset("default"), NLs::HasColumnTableSchemaVersion(1), NLs::HasColumnTableTtlSettingsVersion(1), - NLs::HasColumnTableTtlSettingsTiering("tier1"))); + NLs::HasColumnTableTtlSettingsTiering("Tiering1"))); TString tableSchema4 = R"( Name: "Table4" TtlSettings { Tiering { - EnableTiering : true + UseTiering : "Tiering1" } } )"; @@ -661,7 +661,7 @@ Y_UNIT_TEST_SUITE(TOlap) { Name: "ColumnTable" AlterTtlSettings { Tiering { - EnableTiering : true + UseTiering : "Tiering1" } } )", {NKikimrScheme::StatusInvalidParameter}); diff --git a/ydb/core/tx/tiering/CMakeLists.txt b/ydb/core/tx/tiering/CMakeLists.txt index 0e438b65e9d..b9df41b2f3c 100644 --- a/ydb/core/tx/tiering/CMakeLists.txt +++ b/ydb/core/tx/tiering/CMakeLists.txt @@ -6,7 +6,6 @@ # original buildsystem will not be accepted. -add_subdirectory(common) add_subdirectory(rule) add_subdirectory(tier) add_subdirectory(ut) @@ -21,7 +20,6 @@ target_link_libraries(core-tx-tiering PUBLIC ydb-core-protos tx-tiering-rule tx-tiering-tier - tx-tiering-common core-tablet_flat-protos ydb-core-wrappers api-protos @@ -33,7 +31,6 @@ target_sources(core-tx-tiering PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/tier_cleaner.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/path_cleaner.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/manager.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/snapshot_enrich.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/snapshot.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/s3_actor.cpp ) @@ -48,7 +45,6 @@ target_link_libraries(core-tx-tiering.global PUBLIC ydb-core-protos tx-tiering-rule tx-tiering-tier - tx-tiering-common core-tablet_flat-protos ydb-core-wrappers api-protos diff --git a/ydb/core/tx/tiering/cleaner_task.cpp b/ydb/core/tx/tiering/cleaner_task.cpp index 4973c31bdcc..c7626d71073 100644 --- a/ydb/core/tx/tiering/cleaner_task.cpp +++ b/ydb/core/tx/tiering/cleaner_task.cpp @@ -8,13 +8,13 @@ TTaskCleanerActivity::TFactory::TRegistrator<TTaskCleanerActivity> TTaskCleanerA NKikimrSchemeOp::TTaskCleaner TTaskCleanerActivity::DoSerializeToProto() const { NKikimrSchemeOp::TTaskCleaner result; result.SetPathId(PathId); - result.SetOwnerPath(OwnerPath); + result.SetTieringId(TieringId); return result; } bool TTaskCleanerActivity::DoDeserializeFromProto(const NKikimrSchemeOp::TTaskCleaner& protoData) { PathId = protoData.GetPathId(); - OwnerPath = protoData.GetOwnerPath(); + TieringId = protoData.GetTieringId(); return true; } @@ -22,7 +22,7 @@ void TTaskCleanerActivity::DoExecute(NBackgroundTasks::ITaskExecutorController:: const NBackgroundTasks::TTaskStateContainer& /*state*/) { #ifndef KIKIMR_DISABLE_S3_OPS - TActivationContext::AsActorContext().Register(new TPathCleaner(PathId, OwnerPath, controller)); + TActivationContext::AsActorContext().Register(new TPathCleaner(TieringId, PathId, controller)); #else controller->TaskFinished(); #endif diff --git a/ydb/core/tx/tiering/cleaner_task.h b/ydb/core/tx/tiering/cleaner_task.h index 12e11bb8226..6f4d6a799e0 100644 --- a/ydb/core/tx/tiering/cleaner_task.h +++ b/ydb/core/tx/tiering/cleaner_task.h @@ -8,7 +8,7 @@ namespace NKikimr::NColumnShard::NTiers { class TTaskCleanerActivity: public NBackgroundTasks::IProtoStringSerializable<NKikimrSchemeOp::TTaskCleaner, NBackgroundTasks::ITaskActivity> { private: - YDB_READONLY_DEF(TString, OwnerPath); + YDB_READONLY_DEF(TString, TieringId); YDB_READONLY(ui64, PathId, 0); static TFactory::TRegistrator<TTaskCleanerActivity> Registrator; protected: @@ -20,8 +20,8 @@ protected: public: TTaskCleanerActivity() = default; - TTaskCleanerActivity(const ui64 pathId, const TString& ownerPath) - : OwnerPath(ownerPath) + TTaskCleanerActivity(const TString& tieringId, const ui64 pathId) + : TieringId(tieringId) , PathId(pathId) { diff --git a/ydb/core/tx/tiering/common/CMakeLists.txt b/ydb/core/tx/tiering/common/CMakeLists.txt deleted file mode 100644 index 898b441f818..00000000000 --- a/ydb/core/tx/tiering/common/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ - -# This file was gererated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_library(tx-tiering-common) -target_link_libraries(tx-tiering-common PUBLIC - contrib-libs-cxxsupp - yutil -) -target_sources(tx-tiering-common PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/common/global_tier_id.cpp -) diff --git a/ydb/core/tx/tiering/common/global_tier_id.cpp b/ydb/core/tx/tiering/common/global_tier_id.cpp deleted file mode 100644 index 3bc06dedbf4..00000000000 --- a/ydb/core/tx/tiering/common/global_tier_id.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "global_tier_id.h" - -namespace NKikimr::NColumnShard::NTiers { - -} diff --git a/ydb/core/tx/tiering/common/global_tier_id.h b/ydb/core/tx/tiering/common/global_tier_id.h deleted file mode 100644 index 3675987f684..00000000000 --- a/ydb/core/tx/tiering/common/global_tier_id.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once -#include <ydb/library/accessor/accessor.h> -#include <util/generic/string.h> - -namespace NKikimr::NColumnShard::NTiers { - -class TGlobalTierId { -private: - YDB_ACCESSOR_DEF(TString, OwnerPath); - YDB_ACCESSOR_DEF(TString, TierName); -public: - TGlobalTierId(const TString& ownerPath, const TString& tierName) - : OwnerPath(ownerPath) - , TierName(tierName) { - - } - - bool operator<(const TGlobalTierId& item) const { - return std::tie(OwnerPath, TierName) < std::tie(item.OwnerPath, item.TierName); - } - - TString ToString() const { - return OwnerPath + "." + TierName; - } -}; - -} diff --git a/ydb/core/tx/tiering/external_data.cpp b/ydb/core/tx/tiering/external_data.cpp index a280c35c125..ce5e93cb57d 100644 --- a/ydb/core/tx/tiering/external_data.cpp +++ b/ydb/core/tx/tiering/external_data.cpp @@ -1,5 +1,4 @@ #include "external_data.h" -#include "snapshot_enrich.h" #include <ydb/core/base/path.h> #include <ydb/core/tx/tiering/tier/manager.h> @@ -12,16 +11,11 @@ namespace NKikimr::NColumnShard::NTiers { -void TSnapshotConstructor::EnrichSnapshotData(ISnapshot::TPtr original, NMetadataProvider::ISnapshotAcceptorController::TPtr controller) const -{ - if (!TablesDecoder) { - TablesDecoder = std::make_shared<TTablesDecoderCache>(); - } - TActivationContext::AsActorContext().Register(new TActorSnapshotEnrich(original, controller, TablesDecoder)); +void TSnapshotConstructor::EnrichSnapshotData(ISnapshot::TPtr original, NMetadataProvider::ISnapshotAcceptorController::TPtr controller) const { + controller->Enriched(original); } -TSnapshotConstructor::TSnapshotConstructor() -{ +TSnapshotConstructor::TSnapshotConstructor() { } std::vector<NKikimr::NMetadata::IOperationsManager::TPtr> TSnapshotConstructor::DoGetManagers() const { diff --git a/ydb/core/tx/tiering/external_data.h b/ydb/core/tx/tiering/external_data.h index 4524a792dc8..98ab1dea8e2 100644 --- a/ydb/core/tx/tiering/external_data.h +++ b/ydb/core/tx/tiering/external_data.h @@ -1,6 +1,5 @@ #pragma once #include "snapshot.h" -#include "snapshot_enrich.h" #include <ydb/core/protos/flat_scheme_op.pb.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> @@ -15,7 +14,6 @@ private: using TNavigate = NSchemeCache::TSchemeCacheNavigate; using TBaseActor = TActor<TSnapshotConstructor>; using ISnapshot = NMetadataProvider::ISnapshot; - mutable std::shared_ptr<TTablesDecoderCache> TablesDecoder; protected: virtual std::vector<NMetadata::IOperationsManager::TPtr> DoGetManagers() const override; public: diff --git a/ydb/core/tx/tiering/manager.cpp b/ydb/core/tx/tiering/manager.cpp index 33446f84de6..2aeee9faa2a 100644 --- a/ydb/core/tx/tiering/manager.cpp +++ b/ydb/core/tx/tiering/manager.cpp @@ -3,18 +3,24 @@ #include "s3_actor.h" #include <ydb/core/tx/columnshard/columnshard_private_events.h> +#include <ydb/services/metadata/secret/fetcher.h> namespace NKikimr::NColumnShard { class TTiersManager::TActor: public TActorBootstrapped<TTiersManager::TActor> { private: std::shared_ptr<TTiersManager> Owner; + NMetadataProvider::ISnapshotParser::TPtr SecretsFetcher; + std::shared_ptr<NMetadata::NSecret::TSnapshot> SecretsSnapshot; + std::shared_ptr<NTiers::TConfigsSnapshot> ConfigsSnapshot; TActorId GetExternalDataActorId() const { return NMetadataProvider::MakeServiceId(SelfId().NodeId()); } public: TActor(std::shared_ptr<TTiersManager> owner) - : Owner(owner) { + : Owner(owner) + , SecretsFetcher(std::make_shared<NMetadata::NSecret::TManager>()) + { } ~TActor() { @@ -31,20 +37,34 @@ public: void Bootstrap() { Become(&TThis::StateMain); Send(GetExternalDataActorId(), new NMetadataProvider::TEvSubscribeExternal(Owner->GetExternalDataManipulation())); + Send(GetExternalDataActorId(), new NMetadataProvider::TEvSubscribeExternal(SecretsFetcher)); } void Handle(NMetadataProvider::TEvRefreshSubscriberData::TPtr& ev) { auto snapshot = ev->Get()->GetSnapshot(); - Owner->TakeConfigs(snapshot); + if (auto configs = std::dynamic_pointer_cast<NTiers::TConfigsSnapshot>(snapshot)) { + ConfigsSnapshot = configs; + if (SecretsSnapshot) { + Owner->TakeConfigs(ConfigsSnapshot, SecretsSnapshot); + } + } else if (auto secrets = std::dynamic_pointer_cast<NMetadata::NSecret::TSnapshot>(snapshot)) { + SecretsSnapshot = secrets; + if (ConfigsSnapshot) { + Owner->TakeConfigs(ConfigsSnapshot, SecretsSnapshot); + } + } else { + Y_VERIFY(false, "unexpected behaviour"); + } } void Handle(NActors::TEvents::TEvPoison::TPtr& /*ev*/) { Send(GetExternalDataActorId(), new NMetadataProvider::TEvUnsubscribeExternal(Owner->GetExternalDataManipulation())); + Send(GetExternalDataActorId(), new NMetadataProvider::TEvUnsubscribeExternal(SecretsFetcher)); PassAway(); } }; namespace NTiers { -TManager& TManager::Restart(const TTierConfig& config) { +TManager& TManager::Restart(const TTierConfig& config, std::shared_ptr<NMetadata::NSecret::TSnapshot> secrets) { if (Config.IsSame(config)) { return *this; } @@ -52,7 +72,7 @@ TManager& TManager::Restart(const TTierConfig& config) { Stop(); } Config = config; - Start(); + Start(secrets); return *this; } @@ -67,7 +87,7 @@ bool TManager::Stop() { return true; } -bool TManager::Start() { +bool TManager::Start(std::shared_ptr<NMetadata::NSecret::TSnapshot> secrets) { if (!Config.NeedExport()) { return true; } @@ -79,7 +99,9 @@ bool TManager::Start() { const NActors::TActorId newActor = ctx.Register( CreateS3Actor(TabletId, TabletActorId, Config.GetTierName()) ); - ctx.Send(newActor, new TEvPrivate::TEvS3Settings(Config.GetProtoConfig().GetObjectStorage())); + auto s3Config = Config.GetPatchedConfig(secrets); + + ctx.Send(newActor, new TEvPrivate::TEvS3Settings(s3Config)); Stop(); StorageActorId = newActor; #endif @@ -125,10 +147,11 @@ NKikimr::NOlap::TCompression TManager::ConvertCompression(const NKikimrSchemeOp: } } -void TTiersManager::TakeConfigs(NMetadataProvider::ISnapshot::TPtr snapshotExt) { +void TTiersManager::TakeConfigs(NMetadataProvider::ISnapshot::TPtr snapshotExt, std::shared_ptr<NMetadata::NSecret::TSnapshot> secrets) { auto snapshotPtr = std::dynamic_pointer_cast<NTiers::TConfigsSnapshot>(snapshotExt); Y_VERIFY(snapshotPtr); Snapshot = snapshotExt; + Secrets = secrets; auto& snapshot = *snapshotPtr; for (auto itSelf = Managers.begin(); itSelf != Managers.end(); ) { auto it = snapshot.GetTierConfigs().find(itSelf->first); @@ -136,42 +159,38 @@ void TTiersManager::TakeConfigs(NMetadataProvider::ISnapshot::TPtr snapshotExt) itSelf->second.Stop(); itSelf = Managers.erase(itSelf); } else { - itSelf->second.Restart(it->second); + itSelf->second.Restart(it->second, Secrets); ++itSelf; } } for (auto&& i : snapshot.GetTierConfigs()) { - if (Managers.contains(i.second.GetGlobalTierId())) { + if (Managers.contains(i.second.GetTierName())) { continue; } NTiers::TManager localManager(TabletId, TabletActorId, i.second); - auto& manager = Managers.emplace(i.second.GetGlobalTierId(), std::move(localManager)).first->second; - manager.Start(); + auto& manager = Managers.emplace(i.second.GetTierName(), std::move(localManager)).first->second; + manager.Start(Secrets); } } -TActorId TTiersManager::GetStorageActorId(const NTiers::TGlobalTierId& tierId) { +TActorId TTiersManager::GetStorageActorId(const TString& tierId) { auto it = Managers.find(tierId); if (it == Managers.end()) { - ALS_ERROR(NKikimrServices::TX_TIERING) << "No S3 actor for tier '" << tierId.ToString() << "' at tablet " << TabletId; + ALS_ERROR(NKikimrServices::TX_TIERING) << "No S3 actor for tier '" << tierId << "' at tablet " << TabletId; return {}; } auto actorId = it->second.GetStorageActorId(); if (!actorId) { - ALS_ERROR(NKikimrServices::TX_TIERING) << "Not started storage actor for tier '" << tierId.ToString() << "' at tablet " << TabletId; + ALS_ERROR(NKikimrServices::TX_TIERING) << "Not started storage actor for tier '" << tierId << "' at tablet " << TabletId; return {}; } return actorId; } TTiersManager& TTiersManager::Start(std::shared_ptr<TTiersManager> ownerPtr) { - ALS_ERROR(NKikimrServices::TX_TIERING) << "AAAAAAAAAAAAAA "; Y_VERIFY(!Actor); Actor = new TTiersManager::TActor(ownerPtr); TActivationContext::AsActorContext().RegisterWithSameMailbox(Actor); - for (auto&& i : Managers) { - i.second.Start(); - } return *this; } @@ -189,7 +208,7 @@ TTiersManager& TTiersManager::Stop() { return *this; } -const NTiers::TManager& TTiersManager::GetManagerVerified(const NTiers::TGlobalTierId& tierId) const { +const NTiers::TManager& TTiersManager::GetManagerVerified(const TString& tierId) const { auto it = Managers.find(tierId); Y_VERIFY(it != Managers.end()); return it->second; @@ -209,11 +228,13 @@ THashMap<ui64, NKikimr::NOlap::TTiersInfo> TTiersManager::GetTiering() const { } auto snapshotPtr = std::dynamic_pointer_cast<NTiers::TConfigsSnapshot>(Snapshot); Y_VERIFY(snapshotPtr); - for (auto&& i : snapshotPtr->GetTableTierings()) { - if (!EnabledPathId.contains(i.second.GetTablePathId())) { - continue; + for (auto&& i : PathIdTiering) { + auto* tiering = snapshotPtr->GetTieringById(i.second); + if (!tiering) { + + } else { + result.emplace(i.first, tiering->BuildTiersInfo()); } - result.emplace(i.second.GetTablePathId(), i.second.BuildTiersInfo()); } return result; } diff --git a/ydb/core/tx/tiering/manager.h b/ydb/core/tx/tiering/manager.h index f9effb784c0..171fef7cb28 100644 --- a/ydb/core/tx/tiering/manager.h +++ b/ydb/core/tx/tiering/manager.h @@ -3,7 +3,10 @@ #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/actor.h> + +#include <ydb/services/metadata/secret/snapshot.h> #include <ydb/services/metadata/service.h> + #include <ydb/library/accessor/accessor.h> namespace NKikimr::NColumnShard { @@ -20,25 +23,26 @@ public: static NOlap::TCompression ConvertCompression(const NKikimrSchemeOp::TCompressionOptions& compression); NOlap::TStorageTier BuildTierStorage() const; - TManager& Restart(const TTierConfig& config); + TManager& Restart(const TTierConfig& config, std::shared_ptr<NMetadata::NSecret::TSnapshot> secrets); bool NeedExport() const { return Config.NeedExport(); } bool Stop(); - bool Start(); + bool Start(std::shared_ptr<NMetadata::NSecret::TSnapshot> secrets); }; } class TTiersManager { private: class TActor; - using TManagers = TMap<NTiers::TGlobalTierId, NTiers::TManager>; + using TManagers = std::unordered_map<TString, NTiers::TManager>; ui64 TabletId = 0; const TActorId TabletActorId; TActor* Actor = nullptr; - std::unordered_set<ui64> EnabledPathId; + std::unordered_map<ui64, TString> PathIdTiering; YDB_READONLY_DEF(TManagers, Managers); + std::shared_ptr<NMetadata::NSecret::TSnapshot> Secrets; NMetadataProvider::ISnapshot::TPtr Snapshot; mutable NMetadataProvider::ISnapshotParser::TPtr ExternalDataManipulation; @@ -50,17 +54,17 @@ public: } TActorId GetActorId() const; THashMap<ui64, NOlap::TTiersInfo> GetTiering() const; - void TakeConfigs(NMetadataProvider::ISnapshot::TPtr snapshot); - void EnablePathId(const ui64 pathId) { - EnabledPathId.emplace(pathId); + void TakeConfigs(NMetadataProvider::ISnapshot::TPtr snapshot, std::shared_ptr<NMetadata::NSecret::TSnapshot> secrets); + void EnablePathId(const ui64 pathId, const TString& tieringId) { + PathIdTiering.emplace(pathId, tieringId); } void DisablePathId(const ui64 pathId) { - EnabledPathId.erase(pathId); + PathIdTiering.erase(pathId); } TTiersManager& Start(std::shared_ptr<TTiersManager> ownerPtr); TTiersManager& Stop(); - TActorId GetStorageActorId(const NTiers::TGlobalTierId& tierId); - const NTiers::TManager& GetManagerVerified(const NTiers::TGlobalTierId& tierId) const; + TActorId GetStorageActorId(const TString& tierId); + const NTiers::TManager& GetManagerVerified(const TString& tierId) const; NMetadataProvider::ISnapshotParser::TPtr GetExternalDataManipulation() const; TManagers::const_iterator begin() const { diff --git a/ydb/core/tx/tiering/path_cleaner.cpp b/ydb/core/tx/tiering/path_cleaner.cpp index 7be50ae766e..6fd2a4f0f6f 100644 --- a/ydb/core/tx/tiering/path_cleaner.cpp +++ b/ydb/core/tx/tiering/path_cleaner.cpp @@ -3,6 +3,7 @@ #include "external_data.h" #include <ydb/services/metadata/service.h> +#include <ydb/services/metadata/secret/fetcher.h> namespace NKikimr::NColumnShard::NTiers { @@ -19,36 +20,57 @@ void TPathCleaner::Handle(TEvTierCleared::TPtr& ev) { } NMetadataProvider::ISnapshotParser::TPtr TPathCleaner::GetTieringSnapshotParser() const { - if (!ExternalDataManipulation) { - ExternalDataManipulation = std::make_shared<NTiers::TSnapshotConstructor>(); - } - return ExternalDataManipulation; + return std::make_shared<NTiers::TSnapshotConstructor>(); +} + +NMetadataProvider::ISnapshotParser::TPtr TPathCleaner::GetSecretsSnapshotParser() const { + return std::make_shared<NMetadata::NSecret::TManager>(); } void TPathCleaner::Handle(NMetadataProvider::TEvRefreshSubscriberData::TPtr& ev) { - auto configsSnapshot = ev->Get()->GetSnapshotAs<TConfigsSnapshot>(); - Y_VERIFY(configsSnapshot); - Send(NMetadataProvider::MakeServiceId(SelfId().NodeId()), new NMetadataProvider::TEvUnsubscribeExternal(GetTieringSnapshotParser())); - std::vector<TTierConfig> configs = configsSnapshot->GetTiersForPathId(PathId); - for (auto&& i : configs) { - auto config = NWrappers::IExternalStorageConfig::Construct(i.GetProtoConfig().GetObjectStorage()); - if (!config) { - ALS_ERROR(NKikimrServices::TX_TIERING) << "cannot construct storage config for " << i.GetTierName(); - continue; + if (auto configs = ev->Get()->GetSnapshotPtrAs<TConfigsSnapshot>()) { + Send(NMetadataProvider::MakeServiceId(SelfId().NodeId()), new NMetadataProvider::TEvUnsubscribeExternal(GetTieringSnapshotParser())); + Configs = configs; + } else if (auto secrets = ev->Get()->GetSnapshotPtrAs<NMetadata::NSecret::TSnapshot>()) { + Send(NMetadataProvider::MakeServiceId(SelfId().NodeId()), new NMetadataProvider::TEvUnsubscribeExternal(GetSecretsSnapshotParser())); + Secrets = secrets; + } else { + Y_VERIFY(false); + } + + if (Configs && Secrets) { + const TTieringRule* rule = Configs->GetTieringById(TieringId); + if (!rule) { + ALS_ERROR(NKikimrServices::TX_TIERING) << "cannot detect tiering for " << TieringId; + Controller->TaskFinished(); + return; + } + for (auto&& i : rule->GetIntervals()) { + const auto tier = Configs->GetTierById(i.GetTierName()); + if (!tier) { + ALS_ERROR(NKikimrServices::TX_TIERING) << "cannot detect tiering for " << TieringId; + continue; + } + auto config = NWrappers::IExternalStorageConfig::Construct(tier->GetPatchedConfig(Secrets)); + if (!config) { + ALS_ERROR(NKikimrServices::TX_TIERING) << "cannot construct storage config for " << i.GetTierName(); + continue; + } + Register(new TTierCleaner(i.GetTierName(), SelfId(), PathId, config)); + TiersWait.emplace(i.GetTierName()); } - Register(new TTierCleaner(i.GetTierName(), SelfId(), PathId, config)); - TiersWait.emplace(i.GetTierName()); } } void TPathCleaner::Bootstrap() { Become(&TPathCleaner::StateMain); Send(NMetadataProvider::MakeServiceId(SelfId().NodeId()), new NMetadataProvider::TEvSubscribeExternal(GetTieringSnapshotParser())); + Send(NMetadataProvider::MakeServiceId(SelfId().NodeId()), new NMetadataProvider::TEvSubscribeExternal(GetSecretsSnapshotParser())); } -TPathCleaner::TPathCleaner(const ui64 pathId, const TString& ownerPath, NBackgroundTasks::ITaskExecutorController::TPtr controller) +TPathCleaner::TPathCleaner(const TString& tieringId, const ui64 pathId, NBackgroundTasks::ITaskExecutorController::TPtr controller) : PathId(pathId) - , OwnerPath(ownerPath) + , TieringId(tieringId) , Controller(controller) { } diff --git a/ydb/core/tx/tiering/path_cleaner.h b/ydb/core/tx/tiering/path_cleaner.h index a83cfb479d3..4d86896b03d 100644 --- a/ydb/core/tx/tiering/path_cleaner.h +++ b/ydb/core/tx/tiering/path_cleaner.h @@ -8,25 +8,30 @@ #include <ydb/library/accessor/accessor.h> #include <ydb/services/bg_tasks/abstract/activity.h> #include <ydb/services/metadata/abstract/common.h> +#include <ydb/services/metadata/secret/snapshot.h> #include <library/cpp/actors/core/actor_bootstrapped.h> namespace NKikimr::NColumnShard::NTiers { +class TConfigsSnapshot; class TPathCleaner: public TActorBootstrapped<TPathCleaner> { private: YDB_READONLY(ui64, PathId, 0); - YDB_READONLY_DEF(TString, OwnerPath); + YDB_READONLY_DEF(TString, TieringId); bool Truncated = false; + std::shared_ptr<TConfigsSnapshot> Configs; + std::shared_ptr<NMetadata::NSecret::TSnapshot> Secrets; std::set<TString> TiersWait; NBackgroundTasks::ITaskExecutorController::TPtr Controller; - mutable NMetadataProvider::ISnapshotParser::TPtr ExternalDataManipulation; + NMetadataProvider::ISnapshotParser::TPtr GetTieringSnapshotParser() const; + NMetadataProvider::ISnapshotParser::TPtr GetSecretsSnapshotParser() const; protected: void Handle(NMetadataProvider::TEvRefreshSubscriberData::TPtr& ev); void Handle(TEvTierCleared::TPtr& ev); public: - TPathCleaner(const ui64 pathId, const TString& ownerPath, NBackgroundTasks::ITaskExecutorController::TPtr controller); + TPathCleaner(const TString& tieringId, const ui64 pathId, NBackgroundTasks::ITaskExecutorController::TPtr controller); static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::TX_TIERING_PATH_CLEANER; diff --git a/ydb/core/tx/tiering/rule/initializer.cpp b/ydb/core/tx/tiering/rule/initializer.cpp index be36ce9e9e3..a9514bb0065 100644 --- a/ydb/core/tx/tiering/rule/initializer.cpp +++ b/ydb/core/tx/tiering/rule/initializer.cpp @@ -17,27 +17,12 @@ TVector<NKikimr::NMetadataInitializer::ITableModifier::TPtr> TTierRulesInitializ } { auto& column = *request.add_columns(); - column.set_name("ownerPath"); + column.set_name("defaultColumn"); column.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::STRING); } { auto& column = *request.add_columns(); - column.set_name("tierName"); - column.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::STRING); - } - { - auto& column = *request.add_columns(); - column.set_name("tablePath"); - column.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::STRING); - } - { - auto& column = *request.add_columns(); - column.set_name("durationForEvict"); - column.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::STRING); - } - { - auto& column = *request.add_columns(); - column.set_name("column"); + column.set_name("description"); column.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::STRING); } result.emplace_back(new NMetadataInitializer::TGenericTableModifier<NInternal::NRequest::TDialogCreateTable>(request, "create_rules")); diff --git a/ydb/core/tx/tiering/rule/object.cpp b/ydb/core/tx/tiering/rule/object.cpp index 82c72444cdd..f2f9728d874 100644 --- a/ydb/core/tx/tiering/rule/object.cpp +++ b/ydb/core/tx/tiering/rule/object.cpp @@ -2,14 +2,12 @@ #include <ydb/services/metadata/manager/ydb_value_operator.h> namespace NKikimr::NColumnShard::NTiers { + NJson::TJsonValue TTieringRule::GetDebugJson() const { NJson::TJsonValue result = NJson::JSON_MAP; result.InsertValue(TDecoder::TieringRuleId, TieringRuleId); - result.InsertValue(TDecoder::TierName, TierName); - result.InsertValue(TDecoder::TablePath, TablePath); - result.InsertValue("tablePathId", TablePathId); - result.InsertValue(TDecoder::Column, Column); - result.InsertValue(TDecoder::DurationForEvict, DurationForEvict.ToString()); + result.InsertValue(TDecoder::DefaultColumn, DefaultColumn); + result.InsertValue(TDecoder::Description, SerializeDescriptionToJson()); return result; } @@ -24,14 +22,41 @@ void TTieringRule::AlteringPreparation(std::vector<TTieringRule>&& objects, controller->PreparationFinished(std::move(objects)); } +NJson::TJsonValue TTieringRule::SerializeDescriptionToJson() const { + NJson::TJsonValue result = NJson::JSON_MAP; + auto& jsonRules = result.InsertValue("rules", NJson::JSON_ARRAY); + for (auto&& i : Intervals) { + jsonRules.AppendValue(i.SerializeToJson()); + } + return result; +} + +bool TTieringRule::DeserializeDescriptionFromJson(const NJson::TJsonValue & jsonInfo) { + const NJson::TJsonValue::TArray* rules; + if (!jsonInfo["rules"].GetArrayPointer(&rules)) { + return false; + } + for (auto&& i : *rules) { + TTieringInterval interval; + if (!interval.DeserializeFromJson(i)) { + return false; + } + Intervals.emplace_back(std::move(interval)); + } + std::sort(Intervals.begin(), Intervals.end()); + return true; +} + NKikimr::NMetadataManager::TTableRecord TTieringRule::SerializeToRecord() const { NMetadataManager::TTableRecord result; result.SetColumn(TDecoder::TieringRuleId, NMetadataManager::TYDBValue::Bytes(TieringRuleId)); - result.SetColumn(TDecoder::OwnerPath, NMetadataManager::TYDBValue::Bytes(OwnerPath)); - result.SetColumn(TDecoder::TierName, NMetadataManager::TYDBValue::Bytes(TierName)); - result.SetColumn(TDecoder::TablePath, NMetadataManager::TYDBValue::Bytes(TablePath)); - result.SetColumn(TDecoder::DurationForEvict, NMetadataManager::TYDBValue::Bytes(DurationForEvict.ToString())); - result.SetColumn(TDecoder::Column, NMetadataManager::TYDBValue::Bytes(Column)); + result.SetColumn(TDecoder::DefaultColumn, NMetadataManager::TYDBValue::Bytes(DefaultColumn)); + { + auto jsonDescription = SerializeDescriptionToJson(); + NJsonWriter::TBuf sout; + sout.WriteJsonValue(&jsonDescription, true); + result.SetColumn(TDecoder::Description, NMetadataManager::TYDBValue::Bytes(sout.Str())); + } return result; } @@ -39,21 +64,14 @@ bool TTieringRule::DeserializeFromRecord(const TDecoder& decoder, const Ydb::Val if (!decoder.Read(decoder.GetTieringRuleIdIdx(), TieringRuleId, r)) { return false; } - if (!decoder.Read(decoder.GetOwnerPathIdx(), OwnerPath, r)) { + if (!decoder.Read(decoder.GetDefaultColumnIdx(), DefaultColumn, r)) { return false; } - OwnerPath = TFsPath(OwnerPath).Fix().GetPath(); - if (!decoder.Read(decoder.GetTierNameIdx(), TierName, r)) { + NJson::TJsonValue jsonDescription; + if (!decoder.ReadJson(decoder.GetDescriptionIdx(), jsonDescription, r)) { return false; } - if (!decoder.Read(decoder.GetTablePathIdx(), TablePath, r)) { - return false; - } - TablePath = TFsPath(TablePath).Fix().GetPath(); - if (!decoder.Read(decoder.GetDurationForEvictIdx(), DurationForEvict, r)) { - return false; - } - if (!decoder.Read(decoder.GetColumnIdx(), Column, r)) { + if (!DeserializeDescriptionFromJson(jsonDescription)) { return false; } return true; @@ -64,69 +82,23 @@ NMetadata::TOperationParsingResult TTieringRule::BuildPatchFromSettings(const NY NKikimr::NMetadataManager::TTableRecord result; result.SetColumn(TDecoder::TieringRuleId, NMetadataManager::TYDBValue::Bytes(settings.GetObjectId())); { - auto it = settings.GetFeatures().find(TDecoder::TablePath); - if (it != settings.GetFeatures().end()) { - result.SetColumn(TDecoder::TablePath, NMetadataManager::TYDBValue::Bytes(it->second)); - } - } - { - auto it = settings.GetFeatures().find(TDecoder::TierName); - if (it != settings.GetFeatures().end()) { - result.SetColumn(TDecoder::TierName, NMetadataManager::TYDBValue::Bytes(it->second)); - } - } - { - auto it = settings.GetFeatures().find(TDecoder::OwnerPath); - if (it != settings.GetFeatures().end()) { - result.SetColumn(TDecoder::OwnerPath, NMetadataManager::TYDBValue::Bytes(it->second)); - } - } - { - auto it = settings.GetFeatures().find(TDecoder::DurationForEvict); + auto it = settings.GetFeatures().find(TDecoder::DefaultColumn); if (it != settings.GetFeatures().end()) { - result.SetColumn(TDecoder::DurationForEvict, NMetadataManager::TYDBValue::Bytes(it->second)); + result.SetColumn(TDecoder::DefaultColumn, NMetadataManager::TYDBValue::Bytes(it->second)); } } { - auto it = settings.GetFeatures().find(TDecoder::Column); + auto it = settings.GetFeatures().find(TDecoder::Description); if (it != settings.GetFeatures().end()) { - result.SetColumn(TDecoder::Column, NMetadataManager::TYDBValue::Bytes(it->second)); + result.SetColumn(TDecoder::Description, NMetadataManager::TYDBValue::Bytes(it->second)); } } return result; } -NJson::TJsonValue TTableTiering::GetDebugJson() const { - NJson::TJsonValue result = NJson::JSON_MAP; - result.InsertValue(TTieringRule::TDecoder::TablePath, TablePath); - result.InsertValue("tablePathId", TablePathId); - result.InsertValue(TTieringRule::TDecoder::Column, Column); - auto&& jsonRules = result.InsertValue("rules", NJson::JSON_ARRAY); - for (auto&& i : Rules) { - jsonRules.AppendValue(i.GetDebugJson()); - } - return result; -} - -void TTableTiering::AddRule(TTieringRule&& tr) { - if (Rules.size()) { - Y_VERIFY(Rules.back().GetDurationForEvict() <= tr.GetDurationForEvict()); - if (Column != tr.GetColumn()) { - ALS_ERROR(NKikimrServices::TX_TIERING) << "inconsistency rule column: " << - TablePath << "/" << Column << " != " << tr.GetColumn(); - return; - } - } else { - Column = tr.GetColumn(); - TablePath = tr.GetTablePath(); - TablePathId = tr.GetTablePathId(); - } - Rules.emplace_back(std::move(tr)); -} - -NKikimr::NOlap::TTiersInfo TTableTiering::BuildTiersInfo() const { - NOlap::TTiersInfo result(GetColumn()); - for (auto&& r : Rules) { +NKikimr::NOlap::TTiersInfo TTieringRule::BuildTiersInfo() const { + NOlap::TTiersInfo result(GetDefaultColumn()); + for (auto&& r : Intervals) { result.AddTier(r.GetTierName(), Now() - r.GetDurationForEvict()); } return result; @@ -143,11 +115,8 @@ std::vector<Ydb::Column> TTieringRule::TDecoder::GetColumns() { return { NMetadataManager::TYDBColumn::Bytes(TieringRuleId), - NMetadataManager::TYDBColumn::Bytes(OwnerPath), - NMetadataManager::TYDBColumn::Bytes(TierName), - NMetadataManager::TYDBColumn::Bytes(TablePath), - NMetadataManager::TYDBColumn::Bytes(Column), - NMetadataManager::TYDBColumn::Bytes(DurationForEvict) + NMetadataManager::TYDBColumn::Bytes(DefaultColumn), + NMetadataManager::TYDBColumn::Bytes(Description) }; } diff --git a/ydb/core/tx/tiering/rule/object.h b/ydb/core/tx/tiering/rule/object.h index 8b71c0e376e..0b7f55fbaf6 100644 --- a/ydb/core/tx/tiering/rule/object.h +++ b/ydb/core/tx/tiering/rule/object.h @@ -1,7 +1,6 @@ #pragma once #include <ydb/core/protos/flat_scheme_op.pb.h> #include <ydb/core/tx/columnshard/engines/column_engine.h> -#include <ydb/core/tx/tiering/common/global_tier_id.h> #include <ydb/services/metadata/abstract/decoder.h> #include <ydb/services/metadata/service.h> #include <ydb/services/metadata/manager/object.h> @@ -11,50 +10,75 @@ namespace NKikimr::NColumnShard::NTiers { -class TTieringRule: public NMetadataManager::TObject<TTieringRule> { +class TTieringInterval { private: - YDB_ACCESSOR_DEF(TString, TieringRuleId); - YDB_ACCESSOR_DEF(TString, OwnerPath); YDB_ACCESSOR_DEF(TString, TierName); - YDB_ACCESSOR_DEF(TString, TablePath); - YDB_ACCESSOR(ui64, TablePathId, 0); - YDB_ACCESSOR_DEF(TString, Column); YDB_ACCESSOR_DEF(TDuration, DurationForEvict); public: + TTieringInterval() = default; + TTieringInterval(const TString& name, const TDuration d) + : TierName(name) + , DurationForEvict(d) + { + + } + + bool operator<(const TTieringInterval& item) const { + return DurationForEvict < item.DurationForEvict; + } + + NJson::TJsonValue SerializeToJson() const { + NJson::TJsonValue result; + result.InsertValue("tierName", TierName); + result.InsertValue("durationForEvict", DurationForEvict.ToString()); + return result; + } + + bool DeserializeFromJson(const NJson::TJsonValue& jsonInfo) { + if (!jsonInfo["tierName"].GetString(&TierName)) { + return false; + } + const TString dStr = jsonInfo["durationForEvict"].GetStringRobust(); + if (!TDuration::TryParse(dStr, DurationForEvict)) { + return false; + } + return true; + } +}; + +class TTieringRule: public NMetadataManager::TObject<TTieringRule> { +private: + YDB_ACCESSOR_DEF(TString, TieringRuleId); + YDB_ACCESSOR_DEF(TString, DefaultColumn); + YDB_ACCESSOR_DEF(TVector<TTieringInterval>, Intervals); +protected: + NJson::TJsonValue SerializeDescriptionToJson() const; + bool DeserializeDescriptionFromJson(const NJson::TJsonValue& jsonInfo); +public: + void AddInterval(const TString& name, const TDuration evDuration) { + Intervals.emplace_back(TTieringInterval(name, evDuration)); + } + static TString GetTypeId() { return "TIERING_RULE"; } - TGlobalTierId GetGlobalTierId() const { - return TGlobalTierId(OwnerPath, TierName); - } static TString GetStorageTablePath(); static void AlteringPreparation(std::vector<TTieringRule>&& objects, NMetadataManager::IAlterPreparationController<TTieringRule>::TPtr controller, const NMetadata::IOperationsManager::TModificationContext& context); - bool operator<(const TTieringRule& item) const { - return std::tie(TablePath, TierName, Column, DurationForEvict, TieringRuleId) - < std::tie(item.TablePath, item.TierName, item.Column, item.DurationForEvict, item.TieringRuleId); - } - NJson::TJsonValue GetDebugJson() const; class TDecoder: public NInternal::TDecoderBase { private: YDB_READONLY(i32, TieringRuleIdIdx, -1); - YDB_READONLY(i32, OwnerPathIdx, -1); - YDB_READONLY(i32, TablePathIdx, -1); - YDB_READONLY(i32, DurationForEvictIdx, -1); - YDB_READONLY(i32, TierNameIdx, -1); - YDB_READONLY(i32, ColumnIdx, -1); + YDB_READONLY(i32, DefaultColumnIdx, -1); + YDB_READONLY(i32, DescriptionIdx, -1); public: static inline const TString TieringRuleId = "tieringRuleId"; - static inline const TString OwnerPath = "ownerPath"; - static inline const TString TierName = "tierName"; - static inline const TString TablePath = "tablePath"; - static inline const TString DurationForEvict = "durationForEvict"; - static inline const TString Column = "column"; + static inline const TString DefaultColumn = "defaultColumn"; + static inline const TString Description = "description"; static std::vector<Ydb::Column> GetPKColumns(); static std::vector<Ydb::Column> GetColumns(); @@ -62,37 +86,15 @@ public: TDecoder(const Ydb::ResultSet& rawData) { TieringRuleIdIdx = GetFieldIndex(rawData, TieringRuleId); - OwnerPathIdx = GetFieldIndex(rawData, OwnerPath); - TierNameIdx = GetFieldIndex(rawData, TierName); - TablePathIdx = GetFieldIndex(rawData, TablePath); - DurationForEvictIdx = GetFieldIndex(rawData, DurationForEvict); - ColumnIdx = GetFieldIndex(rawData, Column); + DefaultColumnIdx = GetFieldIndex(rawData, DefaultColumn); + DescriptionIdx = GetFieldIndex(rawData, Description); } }; NKikimr::NMetadataManager::TTableRecord SerializeToRecord() const; bool DeserializeFromRecord(const TDecoder& decoder, const Ydb::Value& r); static NMetadata::TOperationParsingResult BuildPatchFromSettings(const NYql::TObjectSettingsImpl& settings, const NMetadata::IOperationsManager::TModificationContext& context); -}; - -class TTableTiering { -private: - YDB_READONLY_DEF(TString, TablePath); - YDB_READONLY(ui64, TablePathId, 0); - YDB_READONLY_DEF(TString, Column); - YDB_READONLY_DEF(TVector<TTieringRule>, Rules); -public: - void SetTablePathId(const ui64 pathId) { - Y_VERIFY(!TablePathId); - TablePathId = pathId; - for (auto&& r : Rules) { - r.SetTablePathId(pathId); - } - } - NJson::TJsonValue GetDebugJson() const; - void AddRule(TTieringRule&& tr); - - NOlap::TTiersInfo BuildTiersInfo() const; + NKikimr::NOlap::TTiersInfo BuildTiersInfo() const; }; } diff --git a/ydb/core/tx/tiering/snapshot.cpp b/ydb/core/tx/tiering/snapshot.cpp index 25a1abf3463..8997364d093 100644 --- a/ydb/core/tx/tiering/snapshot.cpp +++ b/ydb/core/tx/tiering/snapshot.cpp @@ -20,7 +20,7 @@ bool TConfigsSnapshot::DoDeserializeFromResultSet(const Ydb::Table::ExecuteQuery ALS_ERROR(NKikimrServices::TX_TIERING) << "cannot parse tier config from snapshot"; continue; } - TierConfigs.emplace(config.GetGlobalTierId(), config); + TierConfigs.emplace(config.GetTierName(), config); } } { @@ -36,16 +36,15 @@ bool TConfigsSnapshot::DoDeserializeFromResultSet(const Ydb::Table::ExecuteQuery } rulesLocal.emplace_back(std::move(tr)); } - std::sort(rulesLocal.begin(), rulesLocal.end()); for (auto&& i : rulesLocal) { - TableTierings[i.GetTablePath()].AddRule(std::move(i)); + TableTierings.emplace(i.GetTieringRuleId(), std::move(i)); } } return true; } -std::optional<TTierConfig> TConfigsSnapshot::GetValue(const TGlobalTierId& key) const { - auto it = TierConfigs.find(key); +std::optional<TTierConfig> TConfigsSnapshot::GetTierById(const TString& tierName) const { + auto it = TierConfigs.find(tierName); if (it == TierConfigs.end()) { return {}; } else { @@ -53,14 +52,8 @@ std::optional<TTierConfig> TConfigsSnapshot::GetValue(const TGlobalTierId& key) } } -void TConfigsSnapshot::RemapTablePathToId(const TString& path, const ui64 pathId) { - auto it = TableTierings.find(path); - Y_VERIFY(it != TableTierings.end()); - it->second.SetTablePathId(pathId); -} - -const TTableTiering* TConfigsSnapshot::GetTableTiering(const TString& tablePath) const { - auto it = TableTierings.find(tablePath); +const TTieringRule* TConfigsSnapshot::GetTieringById(const TString& tieringId) const { + auto it = TableTierings.find(tieringId); if (it == TableTierings.end()) { return nullptr; } else { @@ -68,30 +61,11 @@ const TTableTiering* TConfigsSnapshot::GetTableTiering(const TString& tablePath) } } -std::vector<NKikimr::NColumnShard::NTiers::TTierConfig> TConfigsSnapshot::GetTiersForPathId(const ui64 pathId) const { - std::vector<TTierConfig> result; - std::set<TGlobalTierId> readyIds; - for (auto&& i : TableTierings) { - for (auto&& r : i.second.GetRules()) { - if (r.GetTablePathId() == pathId) { - auto it = TierConfigs.find(r.GetGlobalTierId()); - if (it == TierConfigs.end()) { - ALS_ERROR(NKikimrServices::TX_TIERING) << "inconsistency tiering for " << r.GetGlobalTierId().ToString(); - continue; - } else if (readyIds.emplace(r.GetGlobalTierId()).second) { - result.emplace_back(it->second); - } - } - } - } - return result; -} - TString NTiers::TConfigsSnapshot::DoSerializeToString() const { NJson::TJsonValue result = NJson::JSON_MAP; auto& jsonTiers = result.InsertValue("tiers", NJson::JSON_MAP); for (auto&& i : TierConfigs) { - jsonTiers.InsertValue(i.first.ToString(), i.second.GetDebugJson()); + jsonTiers.InsertValue(i.first, i.second.GetDebugJson()); } auto& jsonTiering = result.InsertValue("rules", NJson::JSON_MAP); for (auto&& i : TableTierings) { diff --git a/ydb/core/tx/tiering/snapshot.h b/ydb/core/tx/tiering/snapshot.h index 30f19b359d6..07d2b773706 100644 --- a/ydb/core/tx/tiering/snapshot.h +++ b/ydb/core/tx/tiering/snapshot.h @@ -13,18 +13,16 @@ namespace NKikimr::NColumnShard::NTiers { class TConfigsSnapshot: public NMetadataProvider::ISnapshot { private: using TBase = NMetadataProvider::ISnapshot; - using TConfigsMap = TMap<TGlobalTierId, TTierConfig>; + using TConfigsMap = TMap<TString, TTierConfig>; YDB_ACCESSOR_DEF(TConfigsMap, TierConfigs); - using TTieringMap = TMap<TString, TTableTiering>; + using TTieringMap = TMap<TString, TTieringRule>; YDB_ACCESSOR_DEF(TTieringMap, TableTierings); protected: virtual bool DoDeserializeFromResultSet(const Ydb::Table::ExecuteQueryResult& rawData) override; virtual TString DoSerializeToString() const override; public: - std::vector<TTierConfig> GetTiersForPathId(const ui64 pathId) const; - const TTableTiering* GetTableTiering(const TString& tablePath) const; - void RemapTablePathToId(const TString& path, const ui64 pathId); - std::optional<TTierConfig> GetValue(const TGlobalTierId& key) const; + const TTieringRule* GetTieringById(const TString& tieringId) const; + std::optional<TTierConfig> GetTierById(const TString& tierName) const; using TBase::TBase; }; diff --git a/ydb/core/tx/tiering/snapshot_enrich.cpp b/ydb/core/tx/tiering/snapshot_enrich.cpp deleted file mode 100644 index 3f7a0809f06..00000000000 --- a/ydb/core/tx/tiering/snapshot_enrich.cpp +++ /dev/null @@ -1,65 +0,0 @@ -#include "snapshot_enrich.h" -#include "snapshot.h" - -#include <ydb/core/base/path.h> - -#include <util/string/join.h> - -namespace NKikimr::NColumnShard::NTiers { - -void TActorSnapshotEnrich::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - auto current = std::dynamic_pointer_cast<TConfigsSnapshot>(Original); - Y_VERIFY(current); - auto g = PassAwayGuard(); - - auto* info = ev->Get(); - const auto& request = info->Request; - - if (request->ResultSet.empty()) { - Controller->EnrichProblem("cannot resolve table path to ids"); - return; - } - - for (auto&& i : request->ResultSet) { - const TString path = "/" + JoinSeq("/", i.Path); - switch (i.Status) { - case TNavigate::EStatus::Ok: - Cache->MutableRemapper().emplace(path, i.TableId.PathId.LocalPathId); - current->RemapTablePathToId(path, i.TableId.PathId.LocalPathId); - break; - default: - Controller->EnrichProblem(::ToString(i.Status) + " for '" + path + "'"); - return; - } - } - for (auto&& i : current->GetTableTierings()) { - Y_VERIFY(i.second.GetTablePathId()); - } - Controller->Enriched(Original); -} - -void TActorSnapshotEnrich::Bootstrap() { - Become(&TThis::StateMain); - auto current = std::dynamic_pointer_cast<TConfigsSnapshot>(Original); - Y_VERIFY(current); - const auto& remapper = Cache->GetRemapper(); - auto request = MakeHolder<NSchemeCache::TSchemeCacheNavigate>(); - request->DatabaseName = NKikimr::CanonizePath(AppData()->TenantName); - for (auto&& i : current->GetTableTierings()) { - auto it = remapper.find(i.second.GetTablePath()); - if (it != remapper.end()) { - current->RemapTablePathToId(it->first, it->second); - } else { - auto& entry = request->ResultSet.emplace_back(); - entry.Operation = TNavigate::OpTable; - entry.Path = NKikimr::SplitPath(i.second.GetTablePath()); - } - } - if (request->ResultSet.empty()) { - Controller->Enriched(Original); - } else { - Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); - } -} - -} diff --git a/ydb/core/tx/tiering/snapshot_enrich.h b/ydb/core/tx/tiering/snapshot_enrich.h deleted file mode 100644 index 2a24ba2226a..00000000000 --- a/ydb/core/tx/tiering/snapshot_enrich.h +++ /dev/null @@ -1,53 +0,0 @@ -#pragma once - -#include <ydb/core/protos/flat_scheme_op.pb.h> -#include <ydb/core/tx/scheme_cache/scheme_cache.h> -#include <ydb/services/metadata/service.h> - -#include <library/cpp/json/writer/json_value.h> - -namespace NKikimr::NColumnShard::NTiers { - -class TTablesDecoderCache { -private: - using TRemapper = TMap<TString, ui64>; - YDB_ACCESSOR_DEF(TRemapper, Remapper); -public: - using TPtr = std::shared_ptr<TTablesDecoderCache>; -}; - -class TActorSnapshotEnrich: public TActorBootstrapped<TActorSnapshotEnrich> { -private: - using TBase = TActorBootstrapped<TActorSnapshotEnrich>; - using TNavigate = NSchemeCache::TSchemeCacheNavigate; - NMetadataProvider::ISnapshot::TPtr Original; - NMetadataProvider::ISnapshotAcceptorController::TPtr Controller; - TTablesDecoderCache::TPtr Cache; -private: - void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev); -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::TIERING_SNAPSHOT_ENRICH; - } - - STATEFN(StateMain) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); - default: - break; - } - } - - TActorSnapshotEnrich(NMetadataProvider::ISnapshot::TPtr original, - NMetadataProvider::ISnapshotAcceptorController::TPtr controller, TTablesDecoderCache::TPtr cache) - : Original(original) - , Controller(controller) - , Cache(cache) - { - - } - - void Bootstrap(); -}; - -} diff --git a/ydb/core/tx/tiering/tier/CMakeLists.txt b/ydb/core/tx/tiering/tier/CMakeLists.txt index 523f34fb8c2..8ab14cb66a5 100644 --- a/ydb/core/tx/tiering/tier/CMakeLists.txt +++ b/ydb/core/tx/tiering/tier/CMakeLists.txt @@ -13,6 +13,7 @@ target_link_libraries(tx-tiering-tier PUBLIC yutil services-metadata-initializer services-metadata-abstract + services-metadata-secret ) target_sources(tx-tiering-tier PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/tier/object.cpp @@ -25,6 +26,7 @@ target_link_libraries(tx-tiering-tier.global PUBLIC yutil services-metadata-initializer services-metadata-abstract + services-metadata-secret ) target_sources(tx-tiering-tier.global PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/tier/manager.cpp diff --git a/ydb/core/tx/tiering/tier/initializer.cpp b/ydb/core/tx/tiering/tier/initializer.cpp index 4567b99fef6..d414499460d 100644 --- a/ydb/core/tx/tiering/tier/initializer.cpp +++ b/ydb/core/tx/tiering/tier/initializer.cpp @@ -12,11 +12,6 @@ TVector<NKikimr::NMetadataInitializer::ITableModifier::TPtr> TTiersInitializer:: request.add_primary_key("tierName"); { auto& column = *request.add_columns(); - column.set_name("ownerPath"); - column.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::STRING); - } - { - auto& column = *request.add_columns(); column.set_name("tierName"); column.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::STRING); } diff --git a/ydb/core/tx/tiering/tier/object.cpp b/ydb/core/tx/tiering/tier/object.cpp index 638d2614f7a..c09d02ced04 100644 --- a/ydb/core/tx/tiering/tier/object.cpp +++ b/ydb/core/tx/tiering/tier/object.cpp @@ -1,5 +1,7 @@ #include "object.h" + #include <ydb/services/metadata/manager/ydb_value_operator.h> +#include <ydb/services/metadata/secret/fetcher.h> #include <library/cpp/json/writer/json_value.h> #include <library/cpp/protobuf/json/proto2json.h> @@ -8,7 +10,6 @@ namespace NKikimr::NColumnShard::NTiers { NJson::TJsonValue TTierConfig::GetDebugJson() const { NJson::TJsonValue result = NJson::JSON_MAP; - result.InsertValue(TDecoder::OwnerPath, OwnerPath); result.InsertValue(TDecoder::TierName, TierName); NProtobufJson::Proto2Json(ProtoConfig, result.InsertValue(TDecoder::TierConfig, NJson::JSON_MAP)); return result; @@ -19,10 +20,6 @@ bool TTierConfig::IsSame(const TTierConfig& item) const { } bool TTierConfig::DeserializeFromRecord(const TDecoder& decoder, const Ydb::Value& r) { - if (!decoder.Read(decoder.GetOwnerPathIdx(), OwnerPath, r)) { - return false; - } - OwnerPath = TFsPath(OwnerPath).Fix().GetPath(); if (!decoder.Read(decoder.GetTierNameIdx(), TierName, r)) { return false; } @@ -34,7 +31,6 @@ bool TTierConfig::DeserializeFromRecord(const TDecoder& decoder, const Ydb::Valu NKikimr::NMetadataManager::TTableRecord TTierConfig::SerializeToRecord() const { NMetadataManager::TTableRecord result; - result.SetColumn(TDecoder::OwnerPath, NMetadataManager::TYDBValue::Bytes(OwnerPath)); result.SetColumn(TDecoder::TierName, NMetadataManager::TYDBValue::Bytes(TierName)); result.SetColumn(TDecoder::TierConfig, NMetadataManager::TYDBValue::Bytes(ProtoConfig.DebugString())); return result; @@ -55,12 +51,6 @@ NMetadata::TOperationParsingResult TTierConfig::BuildPatchFromSettings(const NYq NKikimr::NMetadataManager::TTableRecord result; result.SetColumn(TDecoder::TierName, NMetadataManager::TYDBValue::Bytes(settings.GetObjectId())); { - auto it = settings.GetFeatures().find(TDecoder::OwnerPath); - if (it != settings.GetFeatures().end()) { - result.SetColumn(TDecoder::OwnerPath, NMetadataManager::TYDBValue::Bytes(it->second)); - } - } - { auto it = settings.GetFeatures().find(TDecoder::TierConfig); if (it != settings.GetFeatures().end()) { TTierProto proto; @@ -74,6 +64,17 @@ NMetadata::TOperationParsingResult TTierConfig::BuildPatchFromSettings(const NYq return result; } +NKikimrSchemeOp::TS3Settings TTierConfig::GetPatchedConfig( + std::shared_ptr<NMetadata::NSecret::TSnapshot> secrets) const +{ + auto config = ProtoConfig.GetObjectStorage(); + if (secrets) { + secrets->PatchString(*config.MutableAccessKey()); + secrets->PatchString(*config.MutableSecretKey()); + } + return config; +} + std::vector<Ydb::Column> TTierConfig::TDecoder::GetPKColumns() { return { NMetadataManager::TYDBColumn::Bytes(TierName) }; } @@ -81,7 +82,6 @@ std::vector<Ydb::Column> TTierConfig::TDecoder::GetPKColumns() { std::vector<Ydb::Column> TTierConfig::TDecoder::GetColumns() { return { - NMetadataManager::TYDBColumn::Bytes(OwnerPath), NMetadataManager::TYDBColumn::Bytes(TierName), NMetadataManager::TYDBColumn::Bytes(TierConfig) }; diff --git a/ydb/core/tx/tiering/tier/object.h b/ydb/core/tx/tiering/tier/object.h index 3d0aef8392b..bbe3ca1071a 100644 --- a/ydb/core/tx/tiering/tier/object.h +++ b/ydb/core/tx/tiering/tier/object.h @@ -1,6 +1,5 @@ #pragma once #include <ydb/core/protos/flat_scheme_op.pb.h> -#include <ydb/core/tx/tiering/common/global_tier_id.h> #include <ydb/services/metadata/abstract/decoder.h> #include <ydb/services/metadata/manager/preparation_controller.h> #include <ydb/services/metadata/manager/table_record.h> @@ -9,38 +8,39 @@ #include <library/cpp/json/writer/json_value.h> +namespace NKikimr::NMetadata::NSecret { +class TSnapshot; +} + namespace NKikimr::NColumnShard::NTiers { class TTierConfig: public NMetadataManager::TObject<TTierConfig> { private: using TTierProto = NKikimrSchemeOp::TStorageTierConfig; - YDB_ACCESSOR_DEF(TString, OwnerPath); YDB_ACCESSOR_DEF(TString, TierName); YDB_ACCESSOR_DEF(TTierProto, ProtoConfig); public: TTierConfig() = default; - TTierConfig(const TString& ownerPath, const TString& tierName) - : OwnerPath(ownerPath) - , TierName(tierName) + TTierConfig(const TString& tierName) + : TierName(tierName) { } + NKikimrSchemeOp::TS3Settings GetPatchedConfig(std::shared_ptr<NMetadata::NSecret::TSnapshot> secrets) const; + class TDecoder: public NInternal::TDecoderBase { private: - YDB_READONLY(i32, OwnerPathIdx, -1); YDB_READONLY(i32, TierNameIdx, -1); YDB_READONLY(i32, TierConfigIdx, -1); public: - static inline const TString OwnerPath = "ownerPath"; static inline const TString TierName = "tierName"; static inline const TString TierConfig = "tierConfig"; static std::vector<Ydb::Column> GetPKColumns(); static std::vector<Ydb::Column> GetColumns(); static std::vector<TString> GetPKColumnIds(); TDecoder(const Ydb::ResultSet& rawData) { - OwnerPathIdx = GetFieldIndex(rawData, OwnerPath); TierNameIdx = GetFieldIndex(rawData, TierName); TierConfigIdx = GetFieldIndex(rawData, TierConfig); } @@ -55,10 +55,6 @@ public: NMetadataManager::IAlterPreparationController<TTierConfig>::TPtr controller, const NMetadata::IOperationsManager::TModificationContext& context); - TGlobalTierId GetGlobalTierId() const { - return TGlobalTierId(OwnerPath, TierName); - } - bool NeedExport() const { return ProtoConfig.HasObjectStorage(); } diff --git a/ydb/core/tx/tiering/ut/ut_tiers.cpp b/ydb/core/tx/tiering/ut/ut_tiers.cpp index 3c657d30766..8b950972def 100644 --- a/ydb/core/tx/tiering/ut/ut_tiers.cpp +++ b/ydb/core/tx/tiering/ut/ut_tiers.cpp @@ -65,7 +65,7 @@ public: ColumnShardCount: %d TtlSettings: { Tiering: { - EnableTiering: true + UseTiering: "tiering1" } } Sharding { @@ -85,6 +85,28 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { const TString ConfigProtoStr1 = "Name : \"abc1\""; const TString ConfigProtoStr2 = "Name : \"abc2\""; + const TString ConfigTiering1Str = R"({ + "rules" : [ + { + "tierName" : "tier1", + "durationForEvict" : "10d" + }, + { + "tierName" : "tier2", + "durationForEvict" : "20d" + } + ] + })"; + + const TString ConfigTiering2Str = R"({ + "rules" : [ + { + "tierName" : "tier1", + "durationForEvict" : "10d" + } + ] + })"; + class TJsonChecker { private: YDB_ACCESSOR_DEF(TString, Path); @@ -121,7 +143,8 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { YDB_READONLY_FLAG(Found, false); YDB_ACCESSOR(ui32, ExpectedTieringsCount, 1); YDB_ACCESSOR(ui32, ExpectedTiersCount, 1); - using TKeyCheckers = TMap<NTiers::TGlobalTierId, TJsonChecker>; + + using TKeyCheckers = TMap<TString, TJsonChecker>; YDB_ACCESSOR_DEF(TKeyCheckers, Checkers); public: void ResetConditions() { @@ -161,42 +184,31 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { void CheckFound(NMetadataProvider::TEvRefreshSubscriberData* event) { auto snapshot = event->GetSnapshotAs<NTiers::TConfigsSnapshot>(); - Y_VERIFY(!!snapshot); - auto* tInfo = snapshot->GetTableTiering("/Root/olapStore/olapTable"); - if (ExpectedTieringsCount) { - if (!tInfo) { - Cerr << "tiering not found: " << snapshot->SerializeToString() << Endl; - return ; - } - if (tInfo->GetRules().size() != ExpectedTieringsCount) { - Cerr << "TieringsCount incorrect: " << snapshot->SerializeToString() << Endl; - return; - } - } else if (tInfo) { - Cerr << "tiering found but its incorrect: " << snapshot->SerializeToString() << Endl; + if (!snapshot) { return; } - for (auto&& [_, tiering] : snapshot->GetTableTierings()) { - if (tiering.GetTablePathId() == 0) { - Cerr << "PathId not initialized: " << snapshot->SerializeToString() << Endl; - return; - } else if (tiering.GetTablePathId() == InvalidLocalPathId) { - Cerr << "PathId invalid: " << snapshot->SerializeToString() << Endl; - return; - } else { - Cerr << "PathId: " << tiering.GetTablePathId() << Endl; - } + const auto& tierings = snapshot->GetTableTierings(); + if (tierings.size() != ExpectedTieringsCount) { + Cerr << "TieringsCount incorrect: " << snapshot->SerializeToString() << ";expectation=" << ExpectedTieringsCount << Endl; + return; } if (ExpectedTiersCount != snapshot->GetTierConfigs().size()) { - Cerr << "TieringsCount incorrect: " << snapshot->SerializeToString() << Endl; + Cerr << "TiersCount incorrect: " << snapshot->SerializeToString() << ";expectation=" << ExpectedTiersCount << Endl; return; } for (auto&& i : Checkers) { - auto value = snapshot->GetValue(i.first); NJson::TJsonValue jsonData; - NProtobufJson::Proto2Json(value->GetProtoConfig(), jsonData); + if (i.first.StartsWith("TIER.")) { + auto value = snapshot->GetTierById(i.first.substr(5)); + NProtobufJson::Proto2Json(value->GetProtoConfig(), jsonData); + } else if (i.first.StartsWith("TIERING_RULE.")) { + auto value = snapshot->GetTierById(i.first.substr(13)); + NProtobufJson::Proto2Json(value->GetProtoConfig(), jsonData); + } else { + Y_VERIFY(false); + } if (!i.second.Check(jsonData)) { - Cerr << "config value incorrect:" << snapshot->SerializeToString() << ";snapshot_check_path=" << i.first.ToString() << Endl; + Cerr << "config value incorrect:" << snapshot->SerializeToString() << ";snapshot_check_path=" << i.first << Endl; Cerr << "json path incorrect:" << jsonData << ";" << i.second.GetDebugString() << Endl; return; } @@ -259,16 +271,16 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { lHelper.CreateTestOlapTable(); { TTestCSEmulator* emulator = new TTestCSEmulator; - emulator->MutableCheckers().emplace(NTiers::TGlobalTierId("/Root/olapStore", "tier1"), TJsonChecker("Name", "abc")); + emulator->MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "abc")); runtime.Register(emulator); runtime.SimulateSleep(TDuration::Seconds(10)); Cerr << "Initialization finished" << Endl; lHelper.StartSchemaRequest("CREATE OBJECT tier1 ( " - "TYPE TIER) WITH (tierConfig = `" + ConfigProtoStr + "`, ownerPath = `/Root/olapStore`)"); + "TYPE TIER) WITH (tierConfig = `" + ConfigProtoStr + "`)"); lHelper.StartSchemaRequest("CREATE OBJECT tier1 (" - "TYPE TIERING_RULE) WITH (tierName = tier1, tablePath = `/Root/olapStore/olapTable`, " - "ownerPath = `/Root/olapStore`, column = timestamp, durationForEvict = `10d` )"); + "TYPE TIERING_RULE) WITH (tierName = tier1, " + "defaultColumn = timestamp, description = `" + ConfigTiering1Str + "` )"); { const TInstant start = Now(); while (!emulator->IsFound() && Now() - start < TDuration::Seconds(2000)) { @@ -281,7 +293,7 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { "TYPE TIER) SET tierConfig = `" + ConfigProtoStr1 + "`"); emulator->ResetConditions(); - emulator->MutableCheckers().emplace(NTiers::TGlobalTierId("/Root/olapStore", "tier1"), TJsonChecker("Name", "abc1")); + emulator->MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "abc1")); { const TInstant start = Now(); while (!emulator->IsFound() && Now() - start < TDuration::Seconds(2000)) { @@ -349,34 +361,34 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { Cerr << "Initialization finished" << Endl; lHelper.StartSchemaRequest("CREATE OBJECT tier1 ( " - "TYPE TIER) WITH (tierConfig = `" + ConfigProtoStr1 + "`, ownerPath = `/Root/olapStore`)"); - lHelper.StartSchemaRequest("CREATE OBJECT tier1 (" + "TYPE TIER) WITH (tierConfig = `" + ConfigProtoStr1 + "`)"); + lHelper.StartSchemaRequest("CREATE OBJECT tiering1 (" "TYPE TIERING_RULE) " - "WITH (ownerPath = `/Root/olapStore`, tierName = tier1, tablePath = `/Root/olapStore/olapTable`, column = timestamp, durationForEvict = `10d` " + "WITH (tierName = tier1, defaultColumn = timestamp, description = `" + ConfigTiering1Str + "` " ")"); { TTestCSEmulator emulator; - emulator.MutableCheckers().emplace(NTiers::TGlobalTierId("/Root/olapStore", "tier1"), TJsonChecker("Name", "abc1")); + emulator.MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "abc1")); emulator.SetExpectedTieringsCount(1); emulator.SetExpectedTiersCount(1); emulator.CheckRuntime(runtime); } lHelper.StartSchemaRequest("CREATE OBJECT tier2 ( " - "TYPE TIER) WITH (ownerPath = `/Root/olapStore`, tierConfig = `" + ConfigProtoStr2 + "`)"); - lHelper.StartSchemaRequest("CREATE OBJECT tier2 (" - "TYPE TIERING_RULE) WITH (tierName = tier2, tablePath = `/Root/olapStore/olapTable`, ownerPath = `/Root/olapStore`, column = timestamp, durationForEvict = `20d` )"); + "TYPE TIER) WITH (tierConfig = `" + ConfigProtoStr2 + "`)"); + lHelper.StartSchemaRequest("CREATE OBJECT tiering2 (" + "TYPE TIERING_RULE) WITH (tierName = tier2, defaultColumn = timestamp, description = `" + ConfigTiering2Str + "` )"); { TTestCSEmulator emulator; - emulator.MutableCheckers().emplace(NTiers::TGlobalTierId("/Root/olapStore", "tier1"), TJsonChecker("Name", "abc1")); - emulator.MutableCheckers().emplace(NTiers::TGlobalTierId("/Root/olapStore", "tier2"), TJsonChecker("Name", "abc2")); + emulator.MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "abc1")); + emulator.MutableCheckers().emplace("TIER.tier2", TJsonChecker("Name", "abc2")); emulator.SetExpectedTieringsCount(2); emulator.SetExpectedTiersCount(2); emulator.CheckRuntime(runtime); } lHelper.StartSchemaRequest("DROP OBJECT tier2 (TYPE TIER)"); - lHelper.StartSchemaRequest("DROP OBJECT tier2 (TYPE TIERING_RULE)"); + lHelper.StartSchemaRequest("DROP OBJECT tiering2 (TYPE TIERING_RULE)"); { TTestCSEmulator emulator; emulator.SetExpectedTieringsCount(1); @@ -384,7 +396,7 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { emulator.CheckRuntime(runtime); } lHelper.StartSchemaRequest("DROP OBJECT tier1 (TYPE TIER)"); - lHelper.StartSchemaRequest("DROP OBJECT tier1 (TYPE TIERING_RULE)"); + lHelper.StartSchemaRequest("DROP OBJECT tiering1 (TYPE TIERING_RULE)"); { TTestCSEmulator emulator; emulator.SetExpectedTieringsCount(0); @@ -405,8 +417,8 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { VerifySSL: false Endpoint: "storage.cloud-preprod.yandex.net" Bucket: "tiering-test-01" - AccessKey: "..." - SecretKey: "..." + AccessKey: "USId:root@builtin:secretAccessKey" + SecretKey: "USId:root@builtin:secretSecretKey" ProxyHost: "localhost" ProxyPort: 8080 ProxyScheme: HTTP @@ -420,6 +432,8 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { ObjectStorage : { Endpoint: "fake" Bucket: "fake" + AccessKey: "USId:root@builtin:secretAccessKey" + SecretKey: "USId:root@builtin:secretSecretKey" } )"; const TString TierEndpoint = "fake"; @@ -451,36 +465,43 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { auto sender = runtime.AllocateEdgeActor(); server->SetupRootStoragePools(sender); - TLocalHelper lHelper(*server); - lHelper.CreateTestOlapTable("olapTable"); runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_NOTICE); runtime.SetLogPriority(NKikimrServices::TX_COLUMNSHARD, NLog::PRI_DEBUG); runtime.SetLogPriority(NKikimrServices::BG_TASKS, NLog::PRI_DEBUG); // runtime.SetLogPriority(NKikimrServices::TX_PROXY_SCHEME_CACHE, NLog::PRI_DEBUG); - Cerr << "Wait initialization" << Endl; - runtime.SimulateSleep(TDuration::Seconds(20)); - Cerr << "Initialization finished" << Endl; - lHelper.StartSchemaRequest("CREATE OBJECT fakeTier1 ( " - "TYPE TIER) WITH (tierConfig = `" + TierConfigProtoStr + "`, ownerPath = `/Root/olapStore`)"); - lHelper.StartSchemaRequest("CREATE OBJECT fakeTier1 (" - "TYPE TIERING_RULE) WITH (tierName = fakeTier1, ownerPath = `/Root/olapStore`, tablePath = `/Root/olapStore/olapTable`, column = timestamp, durationForEvict = `10d` )"); + TLocalHelper lHelper(*server); + lHelper.StartSchemaRequest("CREATE OBJECT secretAccessKey ( " + "TYPE SECRET) WITH (value = ak)"); + lHelper.StartSchemaRequest("CREATE OBJECT secretSecretKey ( " + "TYPE SECRET) WITH (value = sk)"); + Singleton<NKikimr::NWrappers::NExternalStorage::TFakeExternalStorage>()->SetSecretKey("sk"); + + lHelper.StartSchemaRequest("CREATE OBJECT tier1 ( " + "TYPE TIER) WITH (tierConfig = `" + TierConfigProtoStr + "`)"); + lHelper.StartSchemaRequest("CREATE OBJECT tiering1 (" + "TYPE TIERING_RULE) WITH (tierName = tiering1, defaultColumn = timestamp, description = `" + ConfigTiering1Str + "` )"); - lHelper.StartSchemaRequest("CREATE OBJECT fakeTier2 ( " - "TYPE TIER) WITH (tierConfig = `" + TierConfigProtoStr + "`, ownerPath = `/Root/olapStore`)"); - lHelper.StartSchemaRequest("CREATE OBJECT fakeTier2 (" - "TYPE TIERING_RULE) WITH (tierName = fakeTier2, ownerPath = `/Root/olapStore`, tablePath = `/Root/olapStore/olapTable`, column = timestamp, durationForEvict = `20d` )"); + lHelper.StartSchemaRequest("CREATE OBJECT tier2 ( " + "TYPE TIER) WITH (tierConfig = `" + TierConfigProtoStr + "`)"); + lHelper.StartSchemaRequest("CREATE OBJECT tiering2 (" + "TYPE TIERING_RULE) WITH (tierName = tiering2, defaultColumn = timestamp, description = `" + ConfigTiering2Str + "` )"); { - TTestCSEmulator emulator; - emulator.MutableCheckers().emplace(NTiers::TGlobalTierId("/Root/olapStore", "fakeTier1"), TJsonChecker("Name", "fakeTier")); - emulator.MutableCheckers().emplace(NTiers::TGlobalTierId("/Root/olapStore", "fakeTier2"), TJsonChecker("ObjectStorage.Endpoint", TierEndpoint)); - emulator.SetExpectedTieringsCount(2); - emulator.SetExpectedTiersCount(2); - emulator.CheckRuntime(runtime); + TTestCSEmulator* emulator = new TTestCSEmulator; + runtime.Register(emulator); + emulator->MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "fakeTier")); + emulator->MutableCheckers().emplace("TIER.tier2", TJsonChecker("ObjectStorage.Endpoint", TierEndpoint)); + emulator->SetExpectedTieringsCount(2); + emulator->SetExpectedTiersCount(2); + emulator->CheckRuntime(runtime); } + lHelper.CreateTestOlapTable("olapTable"); + Cerr << "Wait tables" << Endl; + runtime.SimulateSleep(TDuration::Seconds(20)); + Cerr << "Initialization tables" << Endl; Cerr << "Insert..." << Endl; - const TInstant pkStart = Now() - TDuration::Days(15); + const TInstant pkStart = Now() - TDuration::Days(5); ui32 idx = 0; lHelper.SendDataViaActorSystem("/Root/olapStore/olapTable", 0, (pkStart + TDuration::Seconds(2 * idx++)).GetValue(), 2000); { diff --git a/ydb/core/wrappers/abstract.cpp b/ydb/core/wrappers/abstract.cpp index 13c4f1bdddc..c13430b7e87 100644 --- a/ydb/core/wrappers/abstract.cpp +++ b/ydb/core/wrappers/abstract.cpp @@ -12,7 +12,7 @@ IExternalStorageOperator::TPtr IExternalStorageConfig::ConstructStorageOperator( IExternalStorageConfig::TPtr IExternalStorageConfig::Construct(const NKikimrSchemeOp::TS3Settings& settings) { if (settings.GetEndpoint() == "fake") { - return std::make_shared<TFakeExternalStorageConfig>(settings.GetBucket()); + return std::make_shared<TFakeExternalStorageConfig>(settings.GetBucket(), settings.GetSecretKey()); } else { return std::make_shared <TS3ExternalStorageConfig>(settings); } diff --git a/ydb/core/wrappers/fake_storage.h b/ydb/core/wrappers/fake_storage.h index 09c23ee7a2a..cc74a36c86f 100644 --- a/ydb/core/wrappers/fake_storage.h +++ b/ydb/core/wrappers/fake_storage.h @@ -46,6 +46,8 @@ public: class TFakeExternalStorage { private: + YDB_ACCESSOR_DEF(TString, SecretKey); + mutable TMutex Mutex; mutable TMap<TString, TFakeBucketStorage> BucketStorages; TEvListObjectsResponse::TResult BuildListObjectsResult(const TEvListObjectsRequest::TRequest& request) const; @@ -103,55 +105,54 @@ public: class TFakeExternalStorageOperator: public IExternalStorageOperator { private: const TString Bucket; + const TString SecretKey; + + template <class TEvent> + void ExecuteImpl(TEvent& ev) const { + ev->Get()->MutableRequest().WithBucket(Bucket); + Y_VERIFY(SecretKey == Singleton<TFakeExternalStorage>()->GetSecretKey()); + Singleton<TFakeExternalStorage>()->Execute(ev); + } + public: - TFakeExternalStorageOperator(const TString& bucket) + TFakeExternalStorageOperator(const TString& bucket, const TString& secretKey) : Bucket(bucket) + , SecretKey(secretKey) { } virtual void Execute(TEvCheckObjectExistsRequest::TPtr& ev) const override { - ev->Get()->MutableRequest().WithBucket(Bucket); - Singleton<TFakeExternalStorage>()->Execute(ev); + ExecuteImpl(ev); } virtual void Execute(TEvListObjectsRequest::TPtr& ev) const override { - ev->Get()->MutableRequest().WithBucket(Bucket); - Singleton<TFakeExternalStorage>()->Execute(ev); + ExecuteImpl(ev); } virtual void Execute(TEvGetObjectRequest::TPtr& ev) const override { - ev->Get()->MutableRequest().WithBucket(Bucket); - Singleton<TFakeExternalStorage>()->Execute(ev); + ExecuteImpl(ev); } virtual void Execute(TEvHeadObjectRequest::TPtr& ev) const override { - ev->Get()->MutableRequest().WithBucket(Bucket); - Singleton<TFakeExternalStorage>()->Execute(ev); + ExecuteImpl(ev); } virtual void Execute(TEvPutObjectRequest::TPtr& ev) const override { - ev->Get()->MutableRequest().WithBucket(Bucket); - Singleton<TFakeExternalStorage>()->Execute(ev); + ExecuteImpl(ev); } virtual void Execute(TEvDeleteObjectRequest::TPtr& ev) const override { - ev->Get()->MutableRequest().WithBucket(Bucket); - Singleton<TFakeExternalStorage>()->Execute(ev); + ExecuteImpl(ev); } virtual void Execute(TEvDeleteObjectsRequest::TPtr& ev) const override { - ev->Get()->MutableRequest().WithBucket(Bucket); - Singleton<TFakeExternalStorage>()->Execute(ev); + ExecuteImpl(ev); } virtual void Execute(TEvCreateMultipartUploadRequest::TPtr& ev) const override { - ev->Get()->MutableRequest().WithBucket(Bucket); - Singleton<TFakeExternalStorage>()->Execute(ev); + ExecuteImpl(ev); } virtual void Execute(TEvUploadPartRequest::TPtr& ev) const override { - ev->Get()->MutableRequest().WithBucket(Bucket); - Singleton<TFakeExternalStorage>()->Execute(ev); + ExecuteImpl(ev); } virtual void Execute(TEvCompleteMultipartUploadRequest::TPtr& ev) const override { - ev->Get()->MutableRequest().WithBucket(Bucket); - Singleton<TFakeExternalStorage>()->Execute(ev); + ExecuteImpl(ev); } virtual void Execute(TEvAbortMultipartUploadRequest::TPtr& ev) const override { - ev->Get()->MutableRequest().WithBucket(Bucket); - Singleton<TFakeExternalStorage>()->Execute(ev); + ExecuteImpl(ev); } }; } // NKikimr::NWrappers::NExternalStorage diff --git a/ydb/core/wrappers/fake_storage_config.cpp b/ydb/core/wrappers/fake_storage_config.cpp index 0d2f9650f2a..5cfd33e0294 100644 --- a/ydb/core/wrappers/fake_storage_config.cpp +++ b/ydb/core/wrappers/fake_storage_config.cpp @@ -10,7 +10,7 @@ TString TFakeExternalStorageConfig::DoGetStorageId() const { } IExternalStorageOperator::TPtr TFakeExternalStorageConfig::DoConstructStorageOperator() const { - return std::make_shared<TFakeExternalStorageOperator>(Bucket); + return std::make_shared<TFakeExternalStorageOperator>(Bucket, SecretKey); } } diff --git a/ydb/core/wrappers/fake_storage_config.h b/ydb/core/wrappers/fake_storage_config.h index e6ce528a7c5..1b3154de18b 100644 --- a/ydb/core/wrappers/fake_storage_config.h +++ b/ydb/core/wrappers/fake_storage_config.h @@ -14,12 +14,14 @@ namespace NKikimr::NWrappers::NExternalStorage { class TFakeExternalStorageConfig: public IExternalStorageConfig { private: const TString Bucket; + const TString SecretKey; protected: virtual TString DoGetStorageId() const override; virtual IExternalStorageOperator::TPtr DoConstructStorageOperator() const override; public: - TFakeExternalStorageConfig(const TString& bucket) + TFakeExternalStorageConfig(const TString& bucket, const TString& secretKey) : Bucket(bucket) + , SecretKey(secretKey) { } diff --git a/ydb/services/metadata/abstract/common.h b/ydb/services/metadata/abstract/common.h index f3cabd77350..3b71104341a 100644 --- a/ydb/services/metadata/abstract/common.h +++ b/ydb/services/metadata/abstract/common.h @@ -44,6 +44,11 @@ public: } template <class TSnapshot> + std::shared_ptr<TSnapshot> GetSnapshotPtrAs() const { + return std::dynamic_pointer_cast<TSnapshot>(Snapshot); + } + + template <class TSnapshot> std::shared_ptr<TSnapshot> GetValidatedSnapshotAs() const { auto result = dynamic_pointer_cast<TSnapshot>(Snapshot); Y_VERIFY(result); diff --git a/ydb/services/metadata/secret/secret.cpp b/ydb/services/metadata/secret/secret.cpp index ccd45b426e2..353fbb3f8e4 100644 --- a/ydb/services/metadata/secret/secret.cpp +++ b/ydb/services/metadata/secret/secret.cpp @@ -87,4 +87,27 @@ std::vector<TString> TSecret::TDecoder::GetPKColumnIds() { return { OwnerUserId, SecretId }; } +TString TSecretId::SerializeToString() const { + TStringBuilder sb; + sb << "USId:" << OwnerUserId << ":" << SecretId; + return sb; +} + +bool TSecretId::DeserializeFromString(const TString& info) { + static const TString prefix = "USId:"; + if (!info.StartsWith(prefix)) { + return false; + } + TStringBuf sb(info.data(), info.size()); + sb.Skip(prefix.size()); + TStringBuf uId; + TStringBuf sId; + if (!sb.TrySplit(':', uId, sId)) { + return false; + } + OwnerUserId = uId; + SecretId = sId; + return true; +} + } diff --git a/ydb/services/metadata/secret/secret.h b/ydb/services/metadata/secret/secret.h index ca6b653f323..0c7c8b833ce 100644 --- a/ydb/services/metadata/secret/secret.h +++ b/ydb/services/metadata/secret/secret.h @@ -21,6 +21,9 @@ public: { } + bool DeserializeFromString(const TString& info); + TString SerializeToString() const; + bool operator<(const TSecretId& item) const { return std::tie(OwnerUserId, SecretId) < std::tie(item.OwnerUserId, item.SecretId); } diff --git a/ydb/services/metadata/secret/snapshot.cpp b/ydb/services/metadata/secret/snapshot.cpp index f97ba3a3bf4..89568d5ab5a 100644 --- a/ydb/services/metadata/secret/snapshot.cpp +++ b/ydb/services/metadata/secret/snapshot.cpp @@ -44,4 +44,17 @@ TString TSnapshot::DoSerializeToString() const { return sb; } +bool TSnapshot::PatchString(TString& stringForPath) const { + TSecretId sId; + if (!sId.DeserializeFromString(stringForPath)) { + return false; + } + auto it = Secrets.find(sId); + if (it == Secrets.end()) { + return false; + } + stringForPath = it->second.GetValue(); + return true; +} + } diff --git a/ydb/services/metadata/secret/snapshot.h b/ydb/services/metadata/secret/snapshot.h index c25ea23600f..541dc985aef 100644 --- a/ydb/services/metadata/secret/snapshot.h +++ b/ydb/services/metadata/secret/snapshot.h @@ -18,6 +18,8 @@ protected: virtual TString DoSerializeToString() const override; public: using TBase::TBase; + + bool PatchString(TString& stringForPath) const; }; } |