aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorivanmorozov <ivanmorozov@yandex-team.com>2022-11-24 19:07:11 +0300
committerivanmorozov <ivanmorozov@yandex-team.com>2022-11-24 19:07:11 +0300
commit6bb204fd7976045e3c045f7fc7cfdbb836e86d46 (patch)
tree89fc57e7303a82e30fa4f73a5b1fa55639b9e5f0
parent9d83e592bfb14abd1e752700722aaf834685206d (diff)
downloadydb-6bb204fd7976045e3c045f7fc7cfdbb836e86d46.tar.gz
correct tiering model (composite tiering rule + direct tiering link in ss)
-rw-r--r--ydb/core/base/events.h2
-rw-r--r--ydb/core/protos/flat_scheme_op.proto4
-rw-r--r--ydb/core/testlib/common_helper.cpp3
-rw-r--r--ydb/core/tx/columnshard/columnshard.cpp2
-rw-r--r--ydb/core/tx/columnshard/columnshard__init.cpp2
-rw-r--r--ydb/core/tx/columnshard/columnshard__write_index.cpp2
-rw-r--r--ydb/core/tx/columnshard/columnshard_impl.cpp30
-rw-r--r--ydb/core/tx/columnshard/columnshard_impl.h8
-rw-r--r--ydb/core/tx/columnshard/columnshard_schema.h8
-rw-r--r--ydb/core/tx/columnshard/columnshard_ut_common.cpp25
-rw-r--r--ydb/core/tx/columnshard/columnshard_ut_common.h6
-rw-r--r--ydb/core/tx/columnshard/ut_columnshard_schema.cpp10
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h2
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_drop_olap_table.cpp25
-rw-r--r--ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp4
-rw-r--r--ydb/core/tx/schemeshard/ut_olap.cpp8
-rw-r--r--ydb/core/tx/tiering/CMakeLists.txt4
-rw-r--r--ydb/core/tx/tiering/cleaner_task.cpp6
-rw-r--r--ydb/core/tx/tiering/cleaner_task.h6
-rw-r--r--ydb/core/tx/tiering/common/CMakeLists.txt17
-rw-r--r--ydb/core/tx/tiering/common/global_tier_id.cpp5
-rw-r--r--ydb/core/tx/tiering/common/global_tier_id.h27
-rw-r--r--ydb/core/tx/tiering/external_data.cpp12
-rw-r--r--ydb/core/tx/tiering/external_data.h2
-rw-r--r--ydb/core/tx/tiering/manager.cpp67
-rw-r--r--ydb/core/tx/tiering/manager.h24
-rw-r--r--ydb/core/tx/tiering/path_cleaner.cpp56
-rw-r--r--ydb/core/tx/tiering/path_cleaner.h11
-rw-r--r--ydb/core/tx/tiering/rule/initializer.cpp19
-rw-r--r--ydb/core/tx/tiering/rule/object.cpp127
-rw-r--r--ydb/core/tx/tiering/rule/object.h102
-rw-r--r--ydb/core/tx/tiering/snapshot.cpp40
-rw-r--r--ydb/core/tx/tiering/snapshot.h10
-rw-r--r--ydb/core/tx/tiering/snapshot_enrich.cpp65
-rw-r--r--ydb/core/tx/tiering/snapshot_enrich.h53
-rw-r--r--ydb/core/tx/tiering/tier/CMakeLists.txt2
-rw-r--r--ydb/core/tx/tiering/tier/initializer.cpp5
-rw-r--r--ydb/core/tx/tiering/tier/object.cpp26
-rw-r--r--ydb/core/tx/tiering/tier/object.h20
-rw-r--r--ydb/core/tx/tiering/ut/ut_tiers.cpp155
-rw-r--r--ydb/core/wrappers/abstract.cpp2
-rw-r--r--ydb/core/wrappers/fake_storage.h47
-rw-r--r--ydb/core/wrappers/fake_storage_config.cpp2
-rw-r--r--ydb/core/wrappers/fake_storage_config.h4
-rw-r--r--ydb/services/metadata/abstract/common.h5
-rw-r--r--ydb/services/metadata/secret/secret.cpp23
-rw-r--r--ydb/services/metadata/secret/secret.h3
-rw-r--r--ydb/services/metadata/secret/snapshot.cpp13
-rw-r--r--ydb/services/metadata/secret/snapshot.h2
49 files changed, 487 insertions, 616 deletions
diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h
index 7cbd1868460..83da6779e76 100644
--- a/ydb/core/base/events.h
+++ b/ydb/core/base/events.h
@@ -155,7 +155,7 @@ struct TKikimrEvents : TEvents {
ES_METADATA_INITIALIZER,
ES_YDB_AUDIT_LOG,
ES_METADATA_MANAGER,
- ES_METADATA_SECRET
+ ES_METADATA_SECRET,
};
};
diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto
index 1cc87172440..9623776b7b1 100644
--- a/ydb/core/protos/flat_scheme_op.proto
+++ b/ydb/core/protos/flat_scheme_op.proto
@@ -471,7 +471,7 @@ message TColumnDataLifeCycle {
}
message TStorageTiering {
- optional bool EnableTiering = 1[default = false];
+ optional string UseTiering = 1;
}
message TDisabled {
@@ -880,7 +880,7 @@ message TS3Settings {
message TTaskCleaner {
optional uint64 PathId = 1;
optional TS3Settings StorageSettings = 2;
- optional string OwnerPath = 3;
+ optional string TieringId = 3;
}
message TBackupTask {
diff --git a/ydb/core/testlib/common_helper.cpp b/ydb/core/testlib/common_helper.cpp
index eb9a9344447..f8bf0d87ec7 100644
--- a/ydb/core/testlib/common_helper.cpp
+++ b/ydb/core/testlib/common_helper.cpp
@@ -37,7 +37,8 @@ void THelper::StartDataRequest(const TString& request, const bool expectSuccess)
}
void THelper::StartSchemaRequest(const TString& request, const bool expectSuccess) const {
- NYdb::NTable::TTableClient tClient(Server.GetDriver(), NYdb::NTable::TClientSettings().UseQueryCache(false));
+ NYdb::NTable::TTableClient tClient(Server.GetDriver(),
+ NYdb::NTable::TClientSettings().UseQueryCache(false).AuthToken("root@builtin"));
auto expectation = expectSuccess;
tClient.CreateSession().Subscribe([request, expectation](NThreading::TFuture<NYdb::NTable::TCreateSessionResult> f) {
auto session = f.GetValueSync().GetSession();
diff --git a/ydb/core/tx/columnshard/columnshard.cpp b/ydb/core/tx/columnshard/columnshard.cpp
index 460e07b02b6..1c2807c66fb 100644
--- a/ydb/core/tx/columnshard/columnshard.cpp
+++ b/ydb/core/tx/columnshard/columnshard.cpp
@@ -30,7 +30,7 @@ void TColumnShard::SwitchToWork(const TActorContext& ctx) {
CompactionActor = ctx.Register(CreateCompactionActor(TabletID(), ctx.SelfID));
EvictionActor = ctx.Register(CreateEvictionActor(TabletID(), ctx.SelfID));
for (auto&& i : Tables) {
- ActivateTiering(i.first, i.second.TieringEnabled);
+ ActivateTiering(i.first, i.second.TieringUsage);
}
SignalTabletActive(ctx);
}
diff --git a/ydb/core/tx/columnshard/columnshard__init.cpp b/ydb/core/tx/columnshard/columnshard__init.cpp
index d1351ca2487..15ee07d9e55 100644
--- a/ydb/core/tx/columnshard/columnshard__init.cpp
+++ b/ydb/core/tx/columnshard/columnshard__init.cpp
@@ -204,7 +204,7 @@ bool TTxInit::ReadEverything(TTransactionContext& txc, const TActorContext& ctx)
const ui64 pathId = rowset.GetValue<Schema::TableInfo::PathId>();
auto& table = Self->Tables[pathId];
table.PathId = pathId;
- table.TieringEnabled = rowset.GetValue<Schema::TableInfo::TieringEnabled>() == "1";
+ table.TieringUsage = rowset.GetValue<Schema::TableInfo::TieringUsage>();
if (rowset.HaveValue<Schema::TableInfo::DropStep>() &&
rowset.HaveValue<Schema::TableInfo::DropTxId>())
{
diff --git a/ydb/core/tx/columnshard/columnshard__write_index.cpp b/ydb/core/tx/columnshard/columnshard__write_index.cpp
index 22ec60a381e..e6c88ef4cd9 100644
--- a/ydb/core/tx/columnshard/columnshard__write_index.cpp
+++ b/ydb/core/tx/columnshard/columnshard__write_index.cpp
@@ -137,7 +137,7 @@ bool TTxWriteIndex::Execute(TTransactionContext& txc, const TActorContext& ctx)
}
// Mark exported blobs
- auto& tManager = Self->GetTierManagerVerified(NTiers::TGlobalTierId(Self->OwnerPath, tierName));
+ auto& tManager = Self->GetTierManagerVerified(tierName);
if (tManager.NeedExport()) {
for (auto& rec : portionInfo.Records) {
auto& blobId = rec.BlobRange.BlobId;
diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp
index 3aa13a26915..38cdf1da14a 100644
--- a/ydb/core/tx/columnshard/columnshard_impl.cpp
+++ b/ydb/core/tx/columnshard/columnshard_impl.cpp
@@ -509,8 +509,8 @@ void TColumnShard::RunEnsureTable(const NKikimrTxColumnShard::TCreateTable& tabl
Ttl.SetPathTtl(pathId, TTtl::TDescription(ttlInfo));
SetCounter(COUNTER_TABLE_TTLS, Ttl.PathsCount());
} else if (ttlInfo.HasTiering()) {
- table.TieringEnabled = ttlInfo.GetTiering().GetEnableTiering();
- ActivateTiering(pathId, table.TieringEnabled);
+ table.TieringUsage = ttlInfo.GetTiering().GetUseTiering();
+ ActivateTiering(pathId, table.TieringUsage);
}
}
@@ -527,7 +527,7 @@ void TColumnShard::RunEnsureTable(const NKikimrTxColumnShard::TCreateTable& tabl
tableVerProto.SetSchemaPresetVersionAdj(tableProto.GetSchemaPresetVersionAdj());
tableVerProto.SetTtlSettingsPresetVersionAdj(tableProto.GetTtlSettingsPresetVersionAdj());
- Schema::SaveTableInfo(db, table.PathId, table.TieringEnabled);
+ Schema::SaveTableInfo(db, table.PathId, table.TieringUsage);
Schema::SaveTableVersionInfo(db, table.PathId, version, tableVerProto);
SetCounter(COUNTER_TABLES, Tables.size());
} else {
@@ -553,8 +553,8 @@ void TColumnShard::RunAlterTable(const NKikimrTxColumnShard::TAlterTable& alterP
info.SetSchemaPresetId(EnsureSchemaPreset(db, alterProto.GetSchemaPreset(), version));
}
- const bool tieringEnabled = alterProto.HasTtlSettings() && alterProto.GetTtlSettings().HasTiering() && alterProto.GetTtlSettings().GetTiering().GetEnableTiering();
- ActivateTiering(pathId, tieringEnabled);
+ const TString& tieringUsage = alterProto.GetTtlSettings().GetTiering().GetUseTiering();
+ ActivateTiering(pathId, tieringUsage);
if (alterProto.HasTtlSettings() && alterProto.GetTtlSettings().HasEnabled()) {
*info.MutableTtlSettings() = alterProto.GetTtlSettings();
Ttl.SetPathTtl(pathId, TTtl::TDescription(alterProto.GetTtlSettings()));
@@ -563,7 +563,7 @@ void TColumnShard::RunAlterTable(const NKikimrTxColumnShard::TAlterTable& alterP
}
info.SetSchemaPresetVersionAdj(alterProto.GetSchemaPresetVersionAdj());
- Schema::SaveTableInfo(db, table.PathId, tieringEnabled);
+ Schema::SaveTableInfo(db, table.PathId, tieringUsage);
Schema::SaveTableVersionInfo(db, table.PathId, version, info);
}
@@ -993,14 +993,14 @@ void TColumnShard::MapExternBlobs(const TActorContext& /*ctx*/, NOlap::TReadMeta
void TColumnShard::ExportBlobs(const TActorContext& ctx, ui64 exportNo, const TString& tierName,
TEvPrivate::TEvExport::TBlobDataMap&& blobsInfo) const {
- if (auto s3 = GetS3ActorForTier(NTiers::TGlobalTierId(OwnerPath, tierName))) {
+ if (auto s3 = GetS3ActorForTier(tierName)) {
auto event = std::make_unique<TEvPrivate::TEvExport>(exportNo, tierName, s3, std::move(blobsInfo));
ctx.Register(CreateExportActor(TabletID(), ctx.SelfID, event.release()));
}
}
void TColumnShard::ForgetBlobs(const TActorContext& ctx, const TString& tierName, std::vector<NOlap::TEvictedBlob>&& blobs) const {
- if (auto s3 = GetS3ActorForTier(NTiers::TGlobalTierId(OwnerPath, tierName))) {
+ if (auto s3 = GetS3ActorForTier(tierName)) {
auto forget = std::make_unique<TEvPrivate::TEvForget>();
forget->Evicted = std::move(blobs);
ctx.Send(s3, forget.release());
@@ -1009,7 +1009,7 @@ void TColumnShard::ForgetBlobs(const TActorContext& ctx, const TString& tierName
bool TColumnShard::GetExportedBlob(const TActorContext& ctx, TActorId dst, ui64 cookie, const TString& tierName,
NOlap::TEvictedBlob&& evicted, std::vector<NOlap::TBlobRange>&& ranges) {
- if (auto s3 = GetS3ActorForTier(NTiers::TGlobalTierId(OwnerPath, tierName))) {
+ if (auto s3 = GetS3ActorForTier(tierName)) {
auto get = std::make_unique<TEvPrivate::TEvGetExported>();
get->DstActor = dst;
get->DstCookie = cookie;
@@ -1031,7 +1031,7 @@ void TColumnShard::Die(const TActorContext& ctx) {
return IActor::Die(ctx);
}
-TActorId TColumnShard::GetS3ActorForTier(const NTiers::TGlobalTierId& tierId) const {
+TActorId TColumnShard::GetS3ActorForTier(const TString& tierId) const {
if (!Tiers) {
return {};
}
@@ -1040,7 +1040,7 @@ TActorId TColumnShard::GetS3ActorForTier(const NTiers::TGlobalTierId& tierId) co
void TColumnShard::Handle(NMetadataProvider::TEvRefreshSubscriberData::TPtr& ev) {
Y_VERIFY(Tiers);
- Tiers->TakeConfigs(ev->Get()->GetSnapshot());
+ Tiers->TakeConfigs(ev->Get()->GetSnapshot(), nullptr);
}
NOlap::TIndexInfo TColumnShard::GetActualIndexInfo(const bool tiersUsage) const {
@@ -1053,14 +1053,14 @@ NOlap::TIndexInfo TColumnShard::GetActualIndexInfo(const bool tiersUsage) const
return indexInfo;
}
-void TColumnShard::ActivateTiering(const ui64 pathId, const bool enableTiering) {
- if (OwnerPath && !Tiers) {
+void TColumnShard::ActivateTiering(const ui64 pathId, const TString& useTiering) {
+ if (!Tiers) {
Tiers = std::make_shared<TTiersManager>(TabletID(), SelfId());
Tiers->Start(Tiers);
}
if (!!Tiers) {
- if (enableTiering) {
- Tiers->EnablePathId(pathId);
+ if (useTiering) {
+ Tiers->EnablePathId(pathId, useTiering);
} else {
Tiers->DisablePathId(pathId);
}
diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h
index 52057f905f3..dccaba39515 100644
--- a/ydb/core/tx/columnshard/columnshard_impl.h
+++ b/ydb/core/tx/columnshard/columnshard_impl.h
@@ -139,7 +139,7 @@ class TColumnShard
Y_UNUSED(ctx);
}
- const NTiers::TManager& GetTierManagerVerified(const NTiers::TGlobalTierId& tierId) const {
+ const NTiers::TManager& GetTierManagerVerified(const TString& tierId) const {
Y_VERIFY(!!Tiers);
return Tiers->GetManagerVerified(tierId);
}
@@ -177,7 +177,7 @@ class TColumnShard
NOlap::TIndexInfo GetActualIndexInfo(const bool tiersUsage = true) const;
- void ActivateTiering(const ui64 pathId, const bool enableTiering);
+ void ActivateTiering(const ui64 pathId, const TString& useTiering);
protected:
STFUNC(StateInit) {
TRACE_EVENT(NKikimrServices::TX_COLUMNSHARD);
@@ -312,7 +312,7 @@ private:
ui64 PathId;
std::map<TRowVersion, TTableVersionInfo> Versions;
TRowVersion DropVersion = TRowVersion::Max();
- bool TieringEnabled = false;
+ TString TieringUsage;
bool IsDropped() const {
return DropVersion != TRowVersion::Max();
@@ -449,7 +449,7 @@ private:
NOlap::TIndexInfo ConvertSchema(const NKikimrSchemeOp::TColumnTableSchema& schema);
void MapExternBlobs(const TActorContext& ctx, NOlap::TReadMetadata& metadata);
- TActorId GetS3ActorForTier(const NTiers::TGlobalTierId& tierId) const;
+ TActorId GetS3ActorForTier(const TString& tierId) const;
void ExportBlobs(const TActorContext& ctx, ui64 exportNo, const TString& tierName,
TEvPrivate::TEvExport::TBlobDataMap&& blobsInfo) const;
void ForgetBlobs(const TActorContext& ctx, const TString& tierName, std::vector<NOlap::TEvictedBlob>&& blobs) const;
diff --git a/ydb/core/tx/columnshard/columnshard_schema.h b/ydb/core/tx/columnshard/columnshard_schema.h
index 4a7ee6c8268..cf5069a7601 100644
--- a/ydb/core/tx/columnshard/columnshard_schema.h
+++ b/ydb/core/tx/columnshard/columnshard_schema.h
@@ -125,10 +125,10 @@ struct Schema : NIceDb::Schema {
struct PathId : Column<1, NScheme::NTypeIds::Uint64> {};
struct DropStep : Column<2, NScheme::NTypeIds::Uint64> {};
struct DropTxId : Column<3, NScheme::NTypeIds::Uint64> {};
- struct TieringEnabled: Column<4, NScheme::NTypeIds::String> {};
+ struct TieringUsage: Column<4, NScheme::NTypeIds::String> {};
using TKey = TableKey<PathId>;
- using TColumns = TableColumns<PathId, DropStep, DropTxId, TieringEnabled>;
+ using TColumns = TableColumns<PathId, DropStep, DropTxId, TieringUsage>;
};
struct TableVersionInfo : Table<11> {
@@ -364,9 +364,9 @@ struct Schema : NIceDb::Schema {
db.Table<SchemaPresetInfo>().Key(id).Delete();
}
- static void SaveTableInfo(NIceDb::TNiceDb& db, const ui64 pathId, const bool tieringEnabled) {
+ static void SaveTableInfo(NIceDb::TNiceDb& db, const ui64 pathId, const TString tieringUsage) {
db.Table<TableInfo>().Key(pathId).Update(
- NIceDb::TUpdate<TableInfo::TieringEnabled>(tieringEnabled ? "1" : "0")
+ NIceDb::TUpdate<TableInfo::TieringUsage>(tieringUsage)
);
}
diff --git a/ydb/core/tx/columnshard/columnshard_ut_common.cpp b/ydb/core/tx/columnshard/columnshard_ut_common.cpp
index 18f08925e92..2b954e8f32e 100644
--- a/ydb/core/tx/columnshard/columnshard_ut_common.cpp
+++ b/ydb/core/tx/columnshard/columnshard_ut_common.cpp
@@ -291,12 +291,20 @@ TSerializedTableRange MakeTestRange(std::pair<ui64, ui64> range, bool inclusiveF
TConstArrayRef<TCell>(cellsTo), inclusiveTo);
}
-NMetadataProvider::ISnapshot::TPtr TTestSchema::BuildSnapshot(const TTableSpecials& specials, const TString& tablePath, const ui32 tablePathId) {
+NMetadataProvider::ISnapshot::TPtr TTestSchema::BuildSnapshot(const TTableSpecials& specials) {
std::unique_ptr<NColumnShard::NTiers::TConfigsSnapshot> cs(new NColumnShard::NTiers::TConfigsSnapshot(Now()));
+ if (specials.Tiers.empty()) {
+ return cs;
+ }
+ NColumnShard::NTiers::TTieringRule tRule;
+ tRule.SetTieringRuleId("Tiering1");
for (auto&& tier : specials.Tiers) {
+ if (!tRule.GetDefaultColumn()) {
+ tRule.SetDefaultColumn(tier.GetTtlColumn());
+ }
+ Y_VERIFY(tRule.GetDefaultColumn() == tier.GetTtlColumn());
{
NColumnShard::NTiers::TTierConfig tConfig;
- tConfig.SetOwnerPath("/Root/olapStore");
tConfig.SetTierName(tier.Name);
tConfig.MutableProtoConfig().SetName(tier.Name);
auto& cProto = tConfig.MutableProtoConfig();
@@ -309,18 +317,11 @@ NMetadataProvider::ISnapshot::TPtr TTestSchema::BuildSnapshot(const TTableSpecia
if (tier.CompressionLevel) {
cProto.MutableCompression()->SetCompressionLevel(*tier.CompressionLevel);
}
- cs->MutableTierConfigs().emplace(tConfig.GetGlobalTierId(), tConfig);
- }
- {
- NColumnShard::NTiers::TTieringRule tRule;
- tRule.SetOwnerPath("/Root/olapStore");
- tRule.SetDurationForEvict(TDuration::Seconds(tier.GetEvictAfterSecondsUnsafe()));
- tRule.SetTablePath(tablePath).SetTablePathId(tablePathId);
- tRule.SetTierName(tier.Name);
- tRule.SetColumn(tier.GetTtlColumn());
- cs->MutableTableTierings()[tablePath].AddRule(std::move(tRule));
+ cs->MutableTierConfigs().emplace(tConfig.GetTierName(), tConfig);
}
+ tRule.AddInterval(tier.Name, TDuration::Seconds(tier.GetEvictAfterSecondsUnsafe()));
}
+ cs->MutableTableTierings().emplace(tRule.GetTieringRuleId(), tRule);
return cs;
}
diff --git a/ydb/core/tx/columnshard/columnshard_ut_common.h b/ydb/core/tx/columnshard/columnshard_ut_common.h
index a850e153f9d..8c4c76b9cb2 100644
--- a/ydb/core/tx/columnshard/columnshard_ut_common.h
+++ b/ydb/core/tx/columnshard/columnshard_ut_common.h
@@ -206,7 +206,7 @@ struct TTestSchema {
static void InitTiers(const TTableSpecials& specials, NKikimrSchemeOp::TColumnDataLifeCycle* ttlSettings) {
Y_VERIFY(specials.HasTiers());
- ttlSettings->MutableTiering()->SetEnableTiering(true);
+ ttlSettings->MutableTiering()->SetUseTiering("Tiering1");
}
static TString CreateTableTxBody(ui64 pathId, const TVector<std::pair<TString, TTypeInfo>>& columns,
@@ -288,7 +288,7 @@ struct TTestSchema {
enable->SetColumnName(specials.GetTtlColumn());
enable->SetExpireAfterSeconds(specials.GetEvictAfterSecondsUnsafe());
} else if (specials.HasTiers()) {
- ttlSettings->MutableTiering()->SetEnableTiering(true);
+ ttlSettings->MutableTiering()->SetUseTiering("Tiering1");
} else {
ttlSettings->MutableDisabled();
}
@@ -308,7 +308,7 @@ struct TTestSchema {
return out;
}
- static NMetadataProvider::ISnapshot::TPtr BuildSnapshot(const TTableSpecials& specials, const TString& tablePath, const ui32 tablePathId);
+ static NMetadataProvider::ISnapshot::TPtr BuildSnapshot(const TTableSpecials& specials);
static TString CommitTxBody(ui64 metaShard, const TVector<ui64>& writeIds) {
NKikimrTxColumnShard::TCommitTxBody proto;
diff --git a/ydb/core/tx/columnshard/ut_columnshard_schema.cpp b/ydb/core/tx/columnshard/ut_columnshard_schema.cpp
index 21b883ae4c4..177d543072e 100644
--- a/ydb/core/tx/columnshard/ut_columnshard_schema.cpp
+++ b/ydb/core/tx/columnshard/ut_columnshard_schema.cpp
@@ -172,7 +172,7 @@ void TestTtl(bool reboots, bool internal, TTestSchema::TTableSpecials spec = {},
UNIT_ASSERT(ok);
PlanSchemaTx(runtime, sender, {planStep, txId});
if (spec.HasTiers()) {
- ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(spec, "test", tableId));
+ ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(spec));
}
//
@@ -237,7 +237,7 @@ void TestTtl(bool reboots, bool internal, TTestSchema::TTableSpecials spec = {},
UNIT_ASSERT(ok);
PlanSchemaTx(runtime, sender, {planStep, txId});
if (spec.HasTiers()) {
- ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(spec, "test", tableId));
+ ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(spec));
}
if (internal) {
@@ -270,7 +270,7 @@ void TestTtl(bool reboots, bool internal, TTestSchema::TTableSpecials spec = {},
{++planStep, ++txId});
UNIT_ASSERT(ok);
if (spec.HasTiers()) {
- ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(TTestSchema::TTableSpecials(), "test", tableId));
+ ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(TTestSchema::TTableSpecials()));
}
PlanSchemaTx(runtime, sender, {planStep, txId});
@@ -445,7 +445,7 @@ TestTiers(bool reboots, const std::vector<TString>& blobs, const std::vector<TTe
}
PlanSchemaTx(runtime, sender, {planStep, txId});
if (specs[0].Tiers.size()) {
- ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(specs[0], "test", tableId));
+ ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(specs[0]));
}
for (auto& data : blobs) {
@@ -484,7 +484,7 @@ TestTiers(bool reboots, const std::vector<TString>& blobs, const std::vector<TTe
}
}
if (specs[i].Tiers.size()) {
- ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(specs[i], "test", tableId));
+ ProvideTieringSnapshot(runtime, sender, TTestSchema::BuildSnapshot(specs[i]));
}
counter.SetRestartTabletOnPutData(reboots ? 1 : 0);
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h b/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h
index 8b62fff951a..618a5a6d1cd 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_backup_restore_common.h
@@ -188,7 +188,7 @@ public:
Bill(operationId, txState.TargetPathId, shardIdx, ru, context);
}
- // TODO(ilnaz): backward compatability, remove it
+ // TODO(ilnaz): backward compatibility, remove it
if (result.GetSuccess()) {
if (result.HasBytesProcessed()) {
txState.DataTotalSize += result.GetBytesProcessed();
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_olap_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_olap_table.cpp
index 8c83c5470ed..01a2f90ef57 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_olap_table.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_olap_table.cpp
@@ -309,20 +309,25 @@ public:
Y_VERIFY(txState->TxType == TTxState::TxDropColumnTable);
LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
- DebugHint() << " ProgressState"
- << ", at schemeshard: " << ssId);
+ DebugHint() << " ProgressState"
+ << ", at schemeshard: " << ssId);
- if (NBackgroundTasks::TServiceOperator::IsEnabled()) {
- NSchemeShard::TPath path = NSchemeShard::TPath::Init(txState->TargetPathId, context.SS);
- TColumnTableInfo::TPtr tableInfo = context.SS->ColumnTables.at(path.Base()->PathId);
- NSchemeShard::TPath ownerPath = tableInfo->IsStandalone() ? path : path.FindOlapStore();
- Y_VERIFY(path.IsResolved());
- NBackgroundTasks::TTask task(std::make_shared<NColumnShard::NTiers::TTaskCleanerActivity>(txState->TargetPathId.LocalPathId, ownerPath.PathString()), nullptr);
+ if (!NBackgroundTasks::TServiceOperator::IsEnabled()) {
+ return Finish(context);
+ }
+ NSchemeShard::TPath path = NSchemeShard::TPath::Init(txState->TargetPathId, context.SS);
+ TColumnTableInfo::TPtr tableInfo = context.SS->ColumnTables.at(path.Base()->PathId);
+ const TString& tieringId = tableInfo->Description.GetTtlSettings().GetTiering().GetUseTiering();
+ if (!tieringId) {
+ return Finish(context);
+ }
+
+ {
+ NBackgroundTasks::TTask task(std::make_shared<NColumnShard::NTiers::TTaskCleanerActivity>(
+ tieringId, txState->TargetPathId.LocalPathId), nullptr);
task.SetId(OperationId.SerializeToString());
context.SS->SelfId().Send(NBackgroundTasks::MakeServiceId(context.SS->SelfId().NodeId()), new NBackgroundTasks::TEvAddTask(std::move(task)));
return false;
- } else {
- return Finish(context);
}
}
};
diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp
index 7d73cfab229..adbb76e37ab 100644
--- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp
+++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp
@@ -924,13 +924,13 @@ TCheckFunc HasColumnTableTtlSettingsDisabled() {
};
}
-TCheckFunc HasColumnTableTtlSettingsTiering(const TString& /*tierName*/) {
+TCheckFunc HasColumnTableTtlSettingsTiering(const TString& tieringName) {
return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) {
const auto& table = record.GetPathDescription().GetColumnTableDescription();
UNIT_ASSERT(table.HasTtlSettings());
const auto& ttl = table.GetTtlSettings();
UNIT_ASSERT(ttl.HasTiering());
- UNIT_ASSERT(ttl.GetTiering().GetEnableTiering());
+ UNIT_ASSERT_EQUAL(ttl.GetTiering().GetUseTiering(), tieringName);
};
}
diff --git a/ydb/core/tx/schemeshard/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap.cpp
index 6b98e69e8b4..f2a37356b0d 100644
--- a/ydb/core/tx/schemeshard/ut_olap.cpp
+++ b/ydb/core/tx/schemeshard/ut_olap.cpp
@@ -512,7 +512,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
Name: "Table3"
TtlSettings {
Tiering {
- EnableTiering : true
+ UseTiering : "Tiering1"
}
}
)";
@@ -524,13 +524,13 @@ Y_UNIT_TEST_SUITE(TOlap) {
NLs::HasColumnTableSchemaPreset("default"),
NLs::HasColumnTableSchemaVersion(1),
NLs::HasColumnTableTtlSettingsVersion(1),
- NLs::HasColumnTableTtlSettingsTiering("tier1")));
+ NLs::HasColumnTableTtlSettingsTiering("Tiering1")));
TString tableSchema4 = R"(
Name: "Table4"
TtlSettings {
Tiering {
- EnableTiering : true
+ UseTiering : "Tiering1"
}
}
)";
@@ -661,7 +661,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
Name: "ColumnTable"
AlterTtlSettings {
Tiering {
- EnableTiering : true
+ UseTiering : "Tiering1"
}
}
)", {NKikimrScheme::StatusInvalidParameter});
diff --git a/ydb/core/tx/tiering/CMakeLists.txt b/ydb/core/tx/tiering/CMakeLists.txt
index 0e438b65e9d..b9df41b2f3c 100644
--- a/ydb/core/tx/tiering/CMakeLists.txt
+++ b/ydb/core/tx/tiering/CMakeLists.txt
@@ -6,7 +6,6 @@
# original buildsystem will not be accepted.
-add_subdirectory(common)
add_subdirectory(rule)
add_subdirectory(tier)
add_subdirectory(ut)
@@ -21,7 +20,6 @@ target_link_libraries(core-tx-tiering PUBLIC
ydb-core-protos
tx-tiering-rule
tx-tiering-tier
- tx-tiering-common
core-tablet_flat-protos
ydb-core-wrappers
api-protos
@@ -33,7 +31,6 @@ target_sources(core-tx-tiering PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/tier_cleaner.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/path_cleaner.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/manager.cpp
- ${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/snapshot_enrich.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/snapshot.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/s3_actor.cpp
)
@@ -48,7 +45,6 @@ target_link_libraries(core-tx-tiering.global PUBLIC
ydb-core-protos
tx-tiering-rule
tx-tiering-tier
- tx-tiering-common
core-tablet_flat-protos
ydb-core-wrappers
api-protos
diff --git a/ydb/core/tx/tiering/cleaner_task.cpp b/ydb/core/tx/tiering/cleaner_task.cpp
index 4973c31bdcc..c7626d71073 100644
--- a/ydb/core/tx/tiering/cleaner_task.cpp
+++ b/ydb/core/tx/tiering/cleaner_task.cpp
@@ -8,13 +8,13 @@ TTaskCleanerActivity::TFactory::TRegistrator<TTaskCleanerActivity> TTaskCleanerA
NKikimrSchemeOp::TTaskCleaner TTaskCleanerActivity::DoSerializeToProto() const {
NKikimrSchemeOp::TTaskCleaner result;
result.SetPathId(PathId);
- result.SetOwnerPath(OwnerPath);
+ result.SetTieringId(TieringId);
return result;
}
bool TTaskCleanerActivity::DoDeserializeFromProto(const NKikimrSchemeOp::TTaskCleaner& protoData) {
PathId = protoData.GetPathId();
- OwnerPath = protoData.GetOwnerPath();
+ TieringId = protoData.GetTieringId();
return true;
}
@@ -22,7 +22,7 @@ void TTaskCleanerActivity::DoExecute(NBackgroundTasks::ITaskExecutorController::
const NBackgroundTasks::TTaskStateContainer& /*state*/)
{
#ifndef KIKIMR_DISABLE_S3_OPS
- TActivationContext::AsActorContext().Register(new TPathCleaner(PathId, OwnerPath, controller));
+ TActivationContext::AsActorContext().Register(new TPathCleaner(TieringId, PathId, controller));
#else
controller->TaskFinished();
#endif
diff --git a/ydb/core/tx/tiering/cleaner_task.h b/ydb/core/tx/tiering/cleaner_task.h
index 12e11bb8226..6f4d6a799e0 100644
--- a/ydb/core/tx/tiering/cleaner_task.h
+++ b/ydb/core/tx/tiering/cleaner_task.h
@@ -8,7 +8,7 @@ namespace NKikimr::NColumnShard::NTiers {
class TTaskCleanerActivity: public NBackgroundTasks::IProtoStringSerializable<NKikimrSchemeOp::TTaskCleaner, NBackgroundTasks::ITaskActivity> {
private:
- YDB_READONLY_DEF(TString, OwnerPath);
+ YDB_READONLY_DEF(TString, TieringId);
YDB_READONLY(ui64, PathId, 0);
static TFactory::TRegistrator<TTaskCleanerActivity> Registrator;
protected:
@@ -20,8 +20,8 @@ protected:
public:
TTaskCleanerActivity() = default;
- TTaskCleanerActivity(const ui64 pathId, const TString& ownerPath)
- : OwnerPath(ownerPath)
+ TTaskCleanerActivity(const TString& tieringId, const ui64 pathId)
+ : TieringId(tieringId)
, PathId(pathId)
{
diff --git a/ydb/core/tx/tiering/common/CMakeLists.txt b/ydb/core/tx/tiering/common/CMakeLists.txt
deleted file mode 100644
index 898b441f818..00000000000
--- a/ydb/core/tx/tiering/common/CMakeLists.txt
+++ /dev/null
@@ -1,17 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(tx-tiering-common)
-target_link_libraries(tx-tiering-common PUBLIC
- contrib-libs-cxxsupp
- yutil
-)
-target_sources(tx-tiering-common PRIVATE
- ${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/common/global_tier_id.cpp
-)
diff --git a/ydb/core/tx/tiering/common/global_tier_id.cpp b/ydb/core/tx/tiering/common/global_tier_id.cpp
deleted file mode 100644
index 3bc06dedbf4..00000000000
--- a/ydb/core/tx/tiering/common/global_tier_id.cpp
+++ /dev/null
@@ -1,5 +0,0 @@
-#include "global_tier_id.h"
-
-namespace NKikimr::NColumnShard::NTiers {
-
-}
diff --git a/ydb/core/tx/tiering/common/global_tier_id.h b/ydb/core/tx/tiering/common/global_tier_id.h
deleted file mode 100644
index 3675987f684..00000000000
--- a/ydb/core/tx/tiering/common/global_tier_id.h
+++ /dev/null
@@ -1,27 +0,0 @@
-#pragma once
-#include <ydb/library/accessor/accessor.h>
-#include <util/generic/string.h>
-
-namespace NKikimr::NColumnShard::NTiers {
-
-class TGlobalTierId {
-private:
- YDB_ACCESSOR_DEF(TString, OwnerPath);
- YDB_ACCESSOR_DEF(TString, TierName);
-public:
- TGlobalTierId(const TString& ownerPath, const TString& tierName)
- : OwnerPath(ownerPath)
- , TierName(tierName) {
-
- }
-
- bool operator<(const TGlobalTierId& item) const {
- return std::tie(OwnerPath, TierName) < std::tie(item.OwnerPath, item.TierName);
- }
-
- TString ToString() const {
- return OwnerPath + "." + TierName;
- }
-};
-
-}
diff --git a/ydb/core/tx/tiering/external_data.cpp b/ydb/core/tx/tiering/external_data.cpp
index a280c35c125..ce5e93cb57d 100644
--- a/ydb/core/tx/tiering/external_data.cpp
+++ b/ydb/core/tx/tiering/external_data.cpp
@@ -1,5 +1,4 @@
#include "external_data.h"
-#include "snapshot_enrich.h"
#include <ydb/core/base/path.h>
#include <ydb/core/tx/tiering/tier/manager.h>
@@ -12,16 +11,11 @@
namespace NKikimr::NColumnShard::NTiers {
-void TSnapshotConstructor::EnrichSnapshotData(ISnapshot::TPtr original, NMetadataProvider::ISnapshotAcceptorController::TPtr controller) const
-{
- if (!TablesDecoder) {
- TablesDecoder = std::make_shared<TTablesDecoderCache>();
- }
- TActivationContext::AsActorContext().Register(new TActorSnapshotEnrich(original, controller, TablesDecoder));
+void TSnapshotConstructor::EnrichSnapshotData(ISnapshot::TPtr original, NMetadataProvider::ISnapshotAcceptorController::TPtr controller) const {
+ controller->Enriched(original);
}
-TSnapshotConstructor::TSnapshotConstructor()
-{
+TSnapshotConstructor::TSnapshotConstructor() {
}
std::vector<NKikimr::NMetadata::IOperationsManager::TPtr> TSnapshotConstructor::DoGetManagers() const {
diff --git a/ydb/core/tx/tiering/external_data.h b/ydb/core/tx/tiering/external_data.h
index 4524a792dc8..98ab1dea8e2 100644
--- a/ydb/core/tx/tiering/external_data.h
+++ b/ydb/core/tx/tiering/external_data.h
@@ -1,6 +1,5 @@
#pragma once
#include "snapshot.h"
-#include "snapshot_enrich.h"
#include <ydb/core/protos/flat_scheme_op.pb.h>
#include <ydb/core/tx/scheme_cache/scheme_cache.h>
@@ -15,7 +14,6 @@ private:
using TNavigate = NSchemeCache::TSchemeCacheNavigate;
using TBaseActor = TActor<TSnapshotConstructor>;
using ISnapshot = NMetadataProvider::ISnapshot;
- mutable std::shared_ptr<TTablesDecoderCache> TablesDecoder;
protected:
virtual std::vector<NMetadata::IOperationsManager::TPtr> DoGetManagers() const override;
public:
diff --git a/ydb/core/tx/tiering/manager.cpp b/ydb/core/tx/tiering/manager.cpp
index 33446f84de6..2aeee9faa2a 100644
--- a/ydb/core/tx/tiering/manager.cpp
+++ b/ydb/core/tx/tiering/manager.cpp
@@ -3,18 +3,24 @@
#include "s3_actor.h"
#include <ydb/core/tx/columnshard/columnshard_private_events.h>
+#include <ydb/services/metadata/secret/fetcher.h>
namespace NKikimr::NColumnShard {
class TTiersManager::TActor: public TActorBootstrapped<TTiersManager::TActor> {
private:
std::shared_ptr<TTiersManager> Owner;
+ NMetadataProvider::ISnapshotParser::TPtr SecretsFetcher;
+ std::shared_ptr<NMetadata::NSecret::TSnapshot> SecretsSnapshot;
+ std::shared_ptr<NTiers::TConfigsSnapshot> ConfigsSnapshot;
TActorId GetExternalDataActorId() const {
return NMetadataProvider::MakeServiceId(SelfId().NodeId());
}
public:
TActor(std::shared_ptr<TTiersManager> owner)
- : Owner(owner) {
+ : Owner(owner)
+ , SecretsFetcher(std::make_shared<NMetadata::NSecret::TManager>())
+ {
}
~TActor() {
@@ -31,20 +37,34 @@ public:
void Bootstrap() {
Become(&TThis::StateMain);
Send(GetExternalDataActorId(), new NMetadataProvider::TEvSubscribeExternal(Owner->GetExternalDataManipulation()));
+ Send(GetExternalDataActorId(), new NMetadataProvider::TEvSubscribeExternal(SecretsFetcher));
}
void Handle(NMetadataProvider::TEvRefreshSubscriberData::TPtr& ev) {
auto snapshot = ev->Get()->GetSnapshot();
- Owner->TakeConfigs(snapshot);
+ if (auto configs = std::dynamic_pointer_cast<NTiers::TConfigsSnapshot>(snapshot)) {
+ ConfigsSnapshot = configs;
+ if (SecretsSnapshot) {
+ Owner->TakeConfigs(ConfigsSnapshot, SecretsSnapshot);
+ }
+ } else if (auto secrets = std::dynamic_pointer_cast<NMetadata::NSecret::TSnapshot>(snapshot)) {
+ SecretsSnapshot = secrets;
+ if (ConfigsSnapshot) {
+ Owner->TakeConfigs(ConfigsSnapshot, SecretsSnapshot);
+ }
+ } else {
+ Y_VERIFY(false, "unexpected behaviour");
+ }
}
void Handle(NActors::TEvents::TEvPoison::TPtr& /*ev*/) {
Send(GetExternalDataActorId(), new NMetadataProvider::TEvUnsubscribeExternal(Owner->GetExternalDataManipulation()));
+ Send(GetExternalDataActorId(), new NMetadataProvider::TEvUnsubscribeExternal(SecretsFetcher));
PassAway();
}
};
namespace NTiers {
-TManager& TManager::Restart(const TTierConfig& config) {
+TManager& TManager::Restart(const TTierConfig& config, std::shared_ptr<NMetadata::NSecret::TSnapshot> secrets) {
if (Config.IsSame(config)) {
return *this;
}
@@ -52,7 +72,7 @@ TManager& TManager::Restart(const TTierConfig& config) {
Stop();
}
Config = config;
- Start();
+ Start(secrets);
return *this;
}
@@ -67,7 +87,7 @@ bool TManager::Stop() {
return true;
}
-bool TManager::Start() {
+bool TManager::Start(std::shared_ptr<NMetadata::NSecret::TSnapshot> secrets) {
if (!Config.NeedExport()) {
return true;
}
@@ -79,7 +99,9 @@ bool TManager::Start() {
const NActors::TActorId newActor = ctx.Register(
CreateS3Actor(TabletId, TabletActorId, Config.GetTierName())
);
- ctx.Send(newActor, new TEvPrivate::TEvS3Settings(Config.GetProtoConfig().GetObjectStorage()));
+ auto s3Config = Config.GetPatchedConfig(secrets);
+
+ ctx.Send(newActor, new TEvPrivate::TEvS3Settings(s3Config));
Stop();
StorageActorId = newActor;
#endif
@@ -125,10 +147,11 @@ NKikimr::NOlap::TCompression TManager::ConvertCompression(const NKikimrSchemeOp:
}
}
-void TTiersManager::TakeConfigs(NMetadataProvider::ISnapshot::TPtr snapshotExt) {
+void TTiersManager::TakeConfigs(NMetadataProvider::ISnapshot::TPtr snapshotExt, std::shared_ptr<NMetadata::NSecret::TSnapshot> secrets) {
auto snapshotPtr = std::dynamic_pointer_cast<NTiers::TConfigsSnapshot>(snapshotExt);
Y_VERIFY(snapshotPtr);
Snapshot = snapshotExt;
+ Secrets = secrets;
auto& snapshot = *snapshotPtr;
for (auto itSelf = Managers.begin(); itSelf != Managers.end(); ) {
auto it = snapshot.GetTierConfigs().find(itSelf->first);
@@ -136,42 +159,38 @@ void TTiersManager::TakeConfigs(NMetadataProvider::ISnapshot::TPtr snapshotExt)
itSelf->second.Stop();
itSelf = Managers.erase(itSelf);
} else {
- itSelf->second.Restart(it->second);
+ itSelf->second.Restart(it->second, Secrets);
++itSelf;
}
}
for (auto&& i : snapshot.GetTierConfigs()) {
- if (Managers.contains(i.second.GetGlobalTierId())) {
+ if (Managers.contains(i.second.GetTierName())) {
continue;
}
NTiers::TManager localManager(TabletId, TabletActorId, i.second);
- auto& manager = Managers.emplace(i.second.GetGlobalTierId(), std::move(localManager)).first->second;
- manager.Start();
+ auto& manager = Managers.emplace(i.second.GetTierName(), std::move(localManager)).first->second;
+ manager.Start(Secrets);
}
}
-TActorId TTiersManager::GetStorageActorId(const NTiers::TGlobalTierId& tierId) {
+TActorId TTiersManager::GetStorageActorId(const TString& tierId) {
auto it = Managers.find(tierId);
if (it == Managers.end()) {
- ALS_ERROR(NKikimrServices::TX_TIERING) << "No S3 actor for tier '" << tierId.ToString() << "' at tablet " << TabletId;
+ ALS_ERROR(NKikimrServices::TX_TIERING) << "No S3 actor for tier '" << tierId << "' at tablet " << TabletId;
return {};
}
auto actorId = it->second.GetStorageActorId();
if (!actorId) {
- ALS_ERROR(NKikimrServices::TX_TIERING) << "Not started storage actor for tier '" << tierId.ToString() << "' at tablet " << TabletId;
+ ALS_ERROR(NKikimrServices::TX_TIERING) << "Not started storage actor for tier '" << tierId << "' at tablet " << TabletId;
return {};
}
return actorId;
}
TTiersManager& TTiersManager::Start(std::shared_ptr<TTiersManager> ownerPtr) {
- ALS_ERROR(NKikimrServices::TX_TIERING) << "AAAAAAAAAAAAAA ";
Y_VERIFY(!Actor);
Actor = new TTiersManager::TActor(ownerPtr);
TActivationContext::AsActorContext().RegisterWithSameMailbox(Actor);
- for (auto&& i : Managers) {
- i.second.Start();
- }
return *this;
}
@@ -189,7 +208,7 @@ TTiersManager& TTiersManager::Stop() {
return *this;
}
-const NTiers::TManager& TTiersManager::GetManagerVerified(const NTiers::TGlobalTierId& tierId) const {
+const NTiers::TManager& TTiersManager::GetManagerVerified(const TString& tierId) const {
auto it = Managers.find(tierId);
Y_VERIFY(it != Managers.end());
return it->second;
@@ -209,11 +228,13 @@ THashMap<ui64, NKikimr::NOlap::TTiersInfo> TTiersManager::GetTiering() const {
}
auto snapshotPtr = std::dynamic_pointer_cast<NTiers::TConfigsSnapshot>(Snapshot);
Y_VERIFY(snapshotPtr);
- for (auto&& i : snapshotPtr->GetTableTierings()) {
- if (!EnabledPathId.contains(i.second.GetTablePathId())) {
- continue;
+ for (auto&& i : PathIdTiering) {
+ auto* tiering = snapshotPtr->GetTieringById(i.second);
+ if (!tiering) {
+
+ } else {
+ result.emplace(i.first, tiering->BuildTiersInfo());
}
- result.emplace(i.second.GetTablePathId(), i.second.BuildTiersInfo());
}
return result;
}
diff --git a/ydb/core/tx/tiering/manager.h b/ydb/core/tx/tiering/manager.h
index f9effb784c0..171fef7cb28 100644
--- a/ydb/core/tx/tiering/manager.h
+++ b/ydb/core/tx/tiering/manager.h
@@ -3,7 +3,10 @@
#include <library/cpp/actors/core/actor_bootstrapped.h>
#include <library/cpp/actors/core/actor.h>
+
+#include <ydb/services/metadata/secret/snapshot.h>
#include <ydb/services/metadata/service.h>
+
#include <ydb/library/accessor/accessor.h>
namespace NKikimr::NColumnShard {
@@ -20,25 +23,26 @@ public:
static NOlap::TCompression ConvertCompression(const NKikimrSchemeOp::TCompressionOptions& compression);
NOlap::TStorageTier BuildTierStorage() const;
- TManager& Restart(const TTierConfig& config);
+ TManager& Restart(const TTierConfig& config, std::shared_ptr<NMetadata::NSecret::TSnapshot> secrets);
bool NeedExport() const {
return Config.NeedExport();
}
bool Stop();
- bool Start();
+ bool Start(std::shared_ptr<NMetadata::NSecret::TSnapshot> secrets);
};
}
class TTiersManager {
private:
class TActor;
- using TManagers = TMap<NTiers::TGlobalTierId, NTiers::TManager>;
+ using TManagers = std::unordered_map<TString, NTiers::TManager>;
ui64 TabletId = 0;
const TActorId TabletActorId;
TActor* Actor = nullptr;
- std::unordered_set<ui64> EnabledPathId;
+ std::unordered_map<ui64, TString> PathIdTiering;
YDB_READONLY_DEF(TManagers, Managers);
+ std::shared_ptr<NMetadata::NSecret::TSnapshot> Secrets;
NMetadataProvider::ISnapshot::TPtr Snapshot;
mutable NMetadataProvider::ISnapshotParser::TPtr ExternalDataManipulation;
@@ -50,17 +54,17 @@ public:
}
TActorId GetActorId() const;
THashMap<ui64, NOlap::TTiersInfo> GetTiering() const;
- void TakeConfigs(NMetadataProvider::ISnapshot::TPtr snapshot);
- void EnablePathId(const ui64 pathId) {
- EnabledPathId.emplace(pathId);
+ void TakeConfigs(NMetadataProvider::ISnapshot::TPtr snapshot, std::shared_ptr<NMetadata::NSecret::TSnapshot> secrets);
+ void EnablePathId(const ui64 pathId, const TString& tieringId) {
+ PathIdTiering.emplace(pathId, tieringId);
}
void DisablePathId(const ui64 pathId) {
- EnabledPathId.erase(pathId);
+ PathIdTiering.erase(pathId);
}
TTiersManager& Start(std::shared_ptr<TTiersManager> ownerPtr);
TTiersManager& Stop();
- TActorId GetStorageActorId(const NTiers::TGlobalTierId& tierId);
- const NTiers::TManager& GetManagerVerified(const NTiers::TGlobalTierId& tierId) const;
+ TActorId GetStorageActorId(const TString& tierId);
+ const NTiers::TManager& GetManagerVerified(const TString& tierId) const;
NMetadataProvider::ISnapshotParser::TPtr GetExternalDataManipulation() const;
TManagers::const_iterator begin() const {
diff --git a/ydb/core/tx/tiering/path_cleaner.cpp b/ydb/core/tx/tiering/path_cleaner.cpp
index 7be50ae766e..6fd2a4f0f6f 100644
--- a/ydb/core/tx/tiering/path_cleaner.cpp
+++ b/ydb/core/tx/tiering/path_cleaner.cpp
@@ -3,6 +3,7 @@
#include "external_data.h"
#include <ydb/services/metadata/service.h>
+#include <ydb/services/metadata/secret/fetcher.h>
namespace NKikimr::NColumnShard::NTiers {
@@ -19,36 +20,57 @@ void TPathCleaner::Handle(TEvTierCleared::TPtr& ev) {
}
NMetadataProvider::ISnapshotParser::TPtr TPathCleaner::GetTieringSnapshotParser() const {
- if (!ExternalDataManipulation) {
- ExternalDataManipulation = std::make_shared<NTiers::TSnapshotConstructor>();
- }
- return ExternalDataManipulation;
+ return std::make_shared<NTiers::TSnapshotConstructor>();
+}
+
+NMetadataProvider::ISnapshotParser::TPtr TPathCleaner::GetSecretsSnapshotParser() const {
+ return std::make_shared<NMetadata::NSecret::TManager>();
}
void TPathCleaner::Handle(NMetadataProvider::TEvRefreshSubscriberData::TPtr& ev) {
- auto configsSnapshot = ev->Get()->GetSnapshotAs<TConfigsSnapshot>();
- Y_VERIFY(configsSnapshot);
- Send(NMetadataProvider::MakeServiceId(SelfId().NodeId()), new NMetadataProvider::TEvUnsubscribeExternal(GetTieringSnapshotParser()));
- std::vector<TTierConfig> configs = configsSnapshot->GetTiersForPathId(PathId);
- for (auto&& i : configs) {
- auto config = NWrappers::IExternalStorageConfig::Construct(i.GetProtoConfig().GetObjectStorage());
- if (!config) {
- ALS_ERROR(NKikimrServices::TX_TIERING) << "cannot construct storage config for " << i.GetTierName();
- continue;
+ if (auto configs = ev->Get()->GetSnapshotPtrAs<TConfigsSnapshot>()) {
+ Send(NMetadataProvider::MakeServiceId(SelfId().NodeId()), new NMetadataProvider::TEvUnsubscribeExternal(GetTieringSnapshotParser()));
+ Configs = configs;
+ } else if (auto secrets = ev->Get()->GetSnapshotPtrAs<NMetadata::NSecret::TSnapshot>()) {
+ Send(NMetadataProvider::MakeServiceId(SelfId().NodeId()), new NMetadataProvider::TEvUnsubscribeExternal(GetSecretsSnapshotParser()));
+ Secrets = secrets;
+ } else {
+ Y_VERIFY(false);
+ }
+
+ if (Configs && Secrets) {
+ const TTieringRule* rule = Configs->GetTieringById(TieringId);
+ if (!rule) {
+ ALS_ERROR(NKikimrServices::TX_TIERING) << "cannot detect tiering for " << TieringId;
+ Controller->TaskFinished();
+ return;
+ }
+ for (auto&& i : rule->GetIntervals()) {
+ const auto tier = Configs->GetTierById(i.GetTierName());
+ if (!tier) {
+ ALS_ERROR(NKikimrServices::TX_TIERING) << "cannot detect tiering for " << TieringId;
+ continue;
+ }
+ auto config = NWrappers::IExternalStorageConfig::Construct(tier->GetPatchedConfig(Secrets));
+ if (!config) {
+ ALS_ERROR(NKikimrServices::TX_TIERING) << "cannot construct storage config for " << i.GetTierName();
+ continue;
+ }
+ Register(new TTierCleaner(i.GetTierName(), SelfId(), PathId, config));
+ TiersWait.emplace(i.GetTierName());
}
- Register(new TTierCleaner(i.GetTierName(), SelfId(), PathId, config));
- TiersWait.emplace(i.GetTierName());
}
}
void TPathCleaner::Bootstrap() {
Become(&TPathCleaner::StateMain);
Send(NMetadataProvider::MakeServiceId(SelfId().NodeId()), new NMetadataProvider::TEvSubscribeExternal(GetTieringSnapshotParser()));
+ Send(NMetadataProvider::MakeServiceId(SelfId().NodeId()), new NMetadataProvider::TEvSubscribeExternal(GetSecretsSnapshotParser()));
}
-TPathCleaner::TPathCleaner(const ui64 pathId, const TString& ownerPath, NBackgroundTasks::ITaskExecutorController::TPtr controller)
+TPathCleaner::TPathCleaner(const TString& tieringId, const ui64 pathId, NBackgroundTasks::ITaskExecutorController::TPtr controller)
: PathId(pathId)
- , OwnerPath(ownerPath)
+ , TieringId(tieringId)
, Controller(controller) {
}
diff --git a/ydb/core/tx/tiering/path_cleaner.h b/ydb/core/tx/tiering/path_cleaner.h
index a83cfb479d3..4d86896b03d 100644
--- a/ydb/core/tx/tiering/path_cleaner.h
+++ b/ydb/core/tx/tiering/path_cleaner.h
@@ -8,25 +8,30 @@
#include <ydb/library/accessor/accessor.h>
#include <ydb/services/bg_tasks/abstract/activity.h>
#include <ydb/services/metadata/abstract/common.h>
+#include <ydb/services/metadata/secret/snapshot.h>
#include <library/cpp/actors/core/actor_bootstrapped.h>
namespace NKikimr::NColumnShard::NTiers {
+class TConfigsSnapshot;
class TPathCleaner: public TActorBootstrapped<TPathCleaner> {
private:
YDB_READONLY(ui64, PathId, 0);
- YDB_READONLY_DEF(TString, OwnerPath);
+ YDB_READONLY_DEF(TString, TieringId);
bool Truncated = false;
+ std::shared_ptr<TConfigsSnapshot> Configs;
+ std::shared_ptr<NMetadata::NSecret::TSnapshot> Secrets;
std::set<TString> TiersWait;
NBackgroundTasks::ITaskExecutorController::TPtr Controller;
- mutable NMetadataProvider::ISnapshotParser::TPtr ExternalDataManipulation;
+
NMetadataProvider::ISnapshotParser::TPtr GetTieringSnapshotParser() const;
+ NMetadataProvider::ISnapshotParser::TPtr GetSecretsSnapshotParser() const;
protected:
void Handle(NMetadataProvider::TEvRefreshSubscriberData::TPtr& ev);
void Handle(TEvTierCleared::TPtr& ev);
public:
- TPathCleaner(const ui64 pathId, const TString& ownerPath, NBackgroundTasks::ITaskExecutorController::TPtr controller);
+ TPathCleaner(const TString& tieringId, const ui64 pathId, NBackgroundTasks::ITaskExecutorController::TPtr controller);
static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
return NKikimrServices::TActivity::TX_TIERING_PATH_CLEANER;
diff --git a/ydb/core/tx/tiering/rule/initializer.cpp b/ydb/core/tx/tiering/rule/initializer.cpp
index be36ce9e9e3..a9514bb0065 100644
--- a/ydb/core/tx/tiering/rule/initializer.cpp
+++ b/ydb/core/tx/tiering/rule/initializer.cpp
@@ -17,27 +17,12 @@ TVector<NKikimr::NMetadataInitializer::ITableModifier::TPtr> TTierRulesInitializ
}
{
auto& column = *request.add_columns();
- column.set_name("ownerPath");
+ column.set_name("defaultColumn");
column.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::STRING);
}
{
auto& column = *request.add_columns();
- column.set_name("tierName");
- column.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::STRING);
- }
- {
- auto& column = *request.add_columns();
- column.set_name("tablePath");
- column.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::STRING);
- }
- {
- auto& column = *request.add_columns();
- column.set_name("durationForEvict");
- column.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::STRING);
- }
- {
- auto& column = *request.add_columns();
- column.set_name("column");
+ column.set_name("description");
column.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::STRING);
}
result.emplace_back(new NMetadataInitializer::TGenericTableModifier<NInternal::NRequest::TDialogCreateTable>(request, "create_rules"));
diff --git a/ydb/core/tx/tiering/rule/object.cpp b/ydb/core/tx/tiering/rule/object.cpp
index 82c72444cdd..f2f9728d874 100644
--- a/ydb/core/tx/tiering/rule/object.cpp
+++ b/ydb/core/tx/tiering/rule/object.cpp
@@ -2,14 +2,12 @@
#include <ydb/services/metadata/manager/ydb_value_operator.h>
namespace NKikimr::NColumnShard::NTiers {
+
NJson::TJsonValue TTieringRule::GetDebugJson() const {
NJson::TJsonValue result = NJson::JSON_MAP;
result.InsertValue(TDecoder::TieringRuleId, TieringRuleId);
- result.InsertValue(TDecoder::TierName, TierName);
- result.InsertValue(TDecoder::TablePath, TablePath);
- result.InsertValue("tablePathId", TablePathId);
- result.InsertValue(TDecoder::Column, Column);
- result.InsertValue(TDecoder::DurationForEvict, DurationForEvict.ToString());
+ result.InsertValue(TDecoder::DefaultColumn, DefaultColumn);
+ result.InsertValue(TDecoder::Description, SerializeDescriptionToJson());
return result;
}
@@ -24,14 +22,41 @@ void TTieringRule::AlteringPreparation(std::vector<TTieringRule>&& objects,
controller->PreparationFinished(std::move(objects));
}
+NJson::TJsonValue TTieringRule::SerializeDescriptionToJson() const {
+ NJson::TJsonValue result = NJson::JSON_MAP;
+ auto& jsonRules = result.InsertValue("rules", NJson::JSON_ARRAY);
+ for (auto&& i : Intervals) {
+ jsonRules.AppendValue(i.SerializeToJson());
+ }
+ return result;
+}
+
+bool TTieringRule::DeserializeDescriptionFromJson(const NJson::TJsonValue & jsonInfo) {
+ const NJson::TJsonValue::TArray* rules;
+ if (!jsonInfo["rules"].GetArrayPointer(&rules)) {
+ return false;
+ }
+ for (auto&& i : *rules) {
+ TTieringInterval interval;
+ if (!interval.DeserializeFromJson(i)) {
+ return false;
+ }
+ Intervals.emplace_back(std::move(interval));
+ }
+ std::sort(Intervals.begin(), Intervals.end());
+ return true;
+}
+
NKikimr::NMetadataManager::TTableRecord TTieringRule::SerializeToRecord() const {
NMetadataManager::TTableRecord result;
result.SetColumn(TDecoder::TieringRuleId, NMetadataManager::TYDBValue::Bytes(TieringRuleId));
- result.SetColumn(TDecoder::OwnerPath, NMetadataManager::TYDBValue::Bytes(OwnerPath));
- result.SetColumn(TDecoder::TierName, NMetadataManager::TYDBValue::Bytes(TierName));
- result.SetColumn(TDecoder::TablePath, NMetadataManager::TYDBValue::Bytes(TablePath));
- result.SetColumn(TDecoder::DurationForEvict, NMetadataManager::TYDBValue::Bytes(DurationForEvict.ToString()));
- result.SetColumn(TDecoder::Column, NMetadataManager::TYDBValue::Bytes(Column));
+ result.SetColumn(TDecoder::DefaultColumn, NMetadataManager::TYDBValue::Bytes(DefaultColumn));
+ {
+ auto jsonDescription = SerializeDescriptionToJson();
+ NJsonWriter::TBuf sout;
+ sout.WriteJsonValue(&jsonDescription, true);
+ result.SetColumn(TDecoder::Description, NMetadataManager::TYDBValue::Bytes(sout.Str()));
+ }
return result;
}
@@ -39,21 +64,14 @@ bool TTieringRule::DeserializeFromRecord(const TDecoder& decoder, const Ydb::Val
if (!decoder.Read(decoder.GetTieringRuleIdIdx(), TieringRuleId, r)) {
return false;
}
- if (!decoder.Read(decoder.GetOwnerPathIdx(), OwnerPath, r)) {
+ if (!decoder.Read(decoder.GetDefaultColumnIdx(), DefaultColumn, r)) {
return false;
}
- OwnerPath = TFsPath(OwnerPath).Fix().GetPath();
- if (!decoder.Read(decoder.GetTierNameIdx(), TierName, r)) {
+ NJson::TJsonValue jsonDescription;
+ if (!decoder.ReadJson(decoder.GetDescriptionIdx(), jsonDescription, r)) {
return false;
}
- if (!decoder.Read(decoder.GetTablePathIdx(), TablePath, r)) {
- return false;
- }
- TablePath = TFsPath(TablePath).Fix().GetPath();
- if (!decoder.Read(decoder.GetDurationForEvictIdx(), DurationForEvict, r)) {
- return false;
- }
- if (!decoder.Read(decoder.GetColumnIdx(), Column, r)) {
+ if (!DeserializeDescriptionFromJson(jsonDescription)) {
return false;
}
return true;
@@ -64,69 +82,23 @@ NMetadata::TOperationParsingResult TTieringRule::BuildPatchFromSettings(const NY
NKikimr::NMetadataManager::TTableRecord result;
result.SetColumn(TDecoder::TieringRuleId, NMetadataManager::TYDBValue::Bytes(settings.GetObjectId()));
{
- auto it = settings.GetFeatures().find(TDecoder::TablePath);
- if (it != settings.GetFeatures().end()) {
- result.SetColumn(TDecoder::TablePath, NMetadataManager::TYDBValue::Bytes(it->second));
- }
- }
- {
- auto it = settings.GetFeatures().find(TDecoder::TierName);
- if (it != settings.GetFeatures().end()) {
- result.SetColumn(TDecoder::TierName, NMetadataManager::TYDBValue::Bytes(it->second));
- }
- }
- {
- auto it = settings.GetFeatures().find(TDecoder::OwnerPath);
- if (it != settings.GetFeatures().end()) {
- result.SetColumn(TDecoder::OwnerPath, NMetadataManager::TYDBValue::Bytes(it->second));
- }
- }
- {
- auto it = settings.GetFeatures().find(TDecoder::DurationForEvict);
+ auto it = settings.GetFeatures().find(TDecoder::DefaultColumn);
if (it != settings.GetFeatures().end()) {
- result.SetColumn(TDecoder::DurationForEvict, NMetadataManager::TYDBValue::Bytes(it->second));
+ result.SetColumn(TDecoder::DefaultColumn, NMetadataManager::TYDBValue::Bytes(it->second));
}
}
{
- auto it = settings.GetFeatures().find(TDecoder::Column);
+ auto it = settings.GetFeatures().find(TDecoder::Description);
if (it != settings.GetFeatures().end()) {
- result.SetColumn(TDecoder::Column, NMetadataManager::TYDBValue::Bytes(it->second));
+ result.SetColumn(TDecoder::Description, NMetadataManager::TYDBValue::Bytes(it->second));
}
}
return result;
}
-NJson::TJsonValue TTableTiering::GetDebugJson() const {
- NJson::TJsonValue result = NJson::JSON_MAP;
- result.InsertValue(TTieringRule::TDecoder::TablePath, TablePath);
- result.InsertValue("tablePathId", TablePathId);
- result.InsertValue(TTieringRule::TDecoder::Column, Column);
- auto&& jsonRules = result.InsertValue("rules", NJson::JSON_ARRAY);
- for (auto&& i : Rules) {
- jsonRules.AppendValue(i.GetDebugJson());
- }
- return result;
-}
-
-void TTableTiering::AddRule(TTieringRule&& tr) {
- if (Rules.size()) {
- Y_VERIFY(Rules.back().GetDurationForEvict() <= tr.GetDurationForEvict());
- if (Column != tr.GetColumn()) {
- ALS_ERROR(NKikimrServices::TX_TIERING) << "inconsistency rule column: " <<
- TablePath << "/" << Column << " != " << tr.GetColumn();
- return;
- }
- } else {
- Column = tr.GetColumn();
- TablePath = tr.GetTablePath();
- TablePathId = tr.GetTablePathId();
- }
- Rules.emplace_back(std::move(tr));
-}
-
-NKikimr::NOlap::TTiersInfo TTableTiering::BuildTiersInfo() const {
- NOlap::TTiersInfo result(GetColumn());
- for (auto&& r : Rules) {
+NKikimr::NOlap::TTiersInfo TTieringRule::BuildTiersInfo() const {
+ NOlap::TTiersInfo result(GetDefaultColumn());
+ for (auto&& r : Intervals) {
result.AddTier(r.GetTierName(), Now() - r.GetDurationForEvict());
}
return result;
@@ -143,11 +115,8 @@ std::vector<Ydb::Column> TTieringRule::TDecoder::GetColumns() {
return
{
NMetadataManager::TYDBColumn::Bytes(TieringRuleId),
- NMetadataManager::TYDBColumn::Bytes(OwnerPath),
- NMetadataManager::TYDBColumn::Bytes(TierName),
- NMetadataManager::TYDBColumn::Bytes(TablePath),
- NMetadataManager::TYDBColumn::Bytes(Column),
- NMetadataManager::TYDBColumn::Bytes(DurationForEvict)
+ NMetadataManager::TYDBColumn::Bytes(DefaultColumn),
+ NMetadataManager::TYDBColumn::Bytes(Description)
};
}
diff --git a/ydb/core/tx/tiering/rule/object.h b/ydb/core/tx/tiering/rule/object.h
index 8b71c0e376e..0b7f55fbaf6 100644
--- a/ydb/core/tx/tiering/rule/object.h
+++ b/ydb/core/tx/tiering/rule/object.h
@@ -1,7 +1,6 @@
#pragma once
#include <ydb/core/protos/flat_scheme_op.pb.h>
#include <ydb/core/tx/columnshard/engines/column_engine.h>
-#include <ydb/core/tx/tiering/common/global_tier_id.h>
#include <ydb/services/metadata/abstract/decoder.h>
#include <ydb/services/metadata/service.h>
#include <ydb/services/metadata/manager/object.h>
@@ -11,50 +10,75 @@
namespace NKikimr::NColumnShard::NTiers {
-class TTieringRule: public NMetadataManager::TObject<TTieringRule> {
+class TTieringInterval {
private:
- YDB_ACCESSOR_DEF(TString, TieringRuleId);
- YDB_ACCESSOR_DEF(TString, OwnerPath);
YDB_ACCESSOR_DEF(TString, TierName);
- YDB_ACCESSOR_DEF(TString, TablePath);
- YDB_ACCESSOR(ui64, TablePathId, 0);
- YDB_ACCESSOR_DEF(TString, Column);
YDB_ACCESSOR_DEF(TDuration, DurationForEvict);
public:
+ TTieringInterval() = default;
+ TTieringInterval(const TString& name, const TDuration d)
+ : TierName(name)
+ , DurationForEvict(d)
+ {
+
+ }
+
+ bool operator<(const TTieringInterval& item) const {
+ return DurationForEvict < item.DurationForEvict;
+ }
+
+ NJson::TJsonValue SerializeToJson() const {
+ NJson::TJsonValue result;
+ result.InsertValue("tierName", TierName);
+ result.InsertValue("durationForEvict", DurationForEvict.ToString());
+ return result;
+ }
+
+ bool DeserializeFromJson(const NJson::TJsonValue& jsonInfo) {
+ if (!jsonInfo["tierName"].GetString(&TierName)) {
+ return false;
+ }
+ const TString dStr = jsonInfo["durationForEvict"].GetStringRobust();
+ if (!TDuration::TryParse(dStr, DurationForEvict)) {
+ return false;
+ }
+ return true;
+ }
+};
+
+class TTieringRule: public NMetadataManager::TObject<TTieringRule> {
+private:
+ YDB_ACCESSOR_DEF(TString, TieringRuleId);
+ YDB_ACCESSOR_DEF(TString, DefaultColumn);
+ YDB_ACCESSOR_DEF(TVector<TTieringInterval>, Intervals);
+protected:
+ NJson::TJsonValue SerializeDescriptionToJson() const;
+ bool DeserializeDescriptionFromJson(const NJson::TJsonValue& jsonInfo);
+public:
+ void AddInterval(const TString& name, const TDuration evDuration) {
+ Intervals.emplace_back(TTieringInterval(name, evDuration));
+ }
+
static TString GetTypeId() {
return "TIERING_RULE";
}
- TGlobalTierId GetGlobalTierId() const {
- return TGlobalTierId(OwnerPath, TierName);
- }
static TString GetStorageTablePath();
static void AlteringPreparation(std::vector<TTieringRule>&& objects,
NMetadataManager::IAlterPreparationController<TTieringRule>::TPtr controller,
const NMetadata::IOperationsManager::TModificationContext& context);
- bool operator<(const TTieringRule& item) const {
- return std::tie(TablePath, TierName, Column, DurationForEvict, TieringRuleId)
- < std::tie(item.TablePath, item.TierName, item.Column, item.DurationForEvict, item.TieringRuleId);
- }
-
NJson::TJsonValue GetDebugJson() const;
class TDecoder: public NInternal::TDecoderBase {
private:
YDB_READONLY(i32, TieringRuleIdIdx, -1);
- YDB_READONLY(i32, OwnerPathIdx, -1);
- YDB_READONLY(i32, TablePathIdx, -1);
- YDB_READONLY(i32, DurationForEvictIdx, -1);
- YDB_READONLY(i32, TierNameIdx, -1);
- YDB_READONLY(i32, ColumnIdx, -1);
+ YDB_READONLY(i32, DefaultColumnIdx, -1);
+ YDB_READONLY(i32, DescriptionIdx, -1);
public:
static inline const TString TieringRuleId = "tieringRuleId";
- static inline const TString OwnerPath = "ownerPath";
- static inline const TString TierName = "tierName";
- static inline const TString TablePath = "tablePath";
- static inline const TString DurationForEvict = "durationForEvict";
- static inline const TString Column = "column";
+ static inline const TString DefaultColumn = "defaultColumn";
+ static inline const TString Description = "description";
static std::vector<Ydb::Column> GetPKColumns();
static std::vector<Ydb::Column> GetColumns();
@@ -62,37 +86,15 @@ public:
TDecoder(const Ydb::ResultSet& rawData) {
TieringRuleIdIdx = GetFieldIndex(rawData, TieringRuleId);
- OwnerPathIdx = GetFieldIndex(rawData, OwnerPath);
- TierNameIdx = GetFieldIndex(rawData, TierName);
- TablePathIdx = GetFieldIndex(rawData, TablePath);
- DurationForEvictIdx = GetFieldIndex(rawData, DurationForEvict);
- ColumnIdx = GetFieldIndex(rawData, Column);
+ DefaultColumnIdx = GetFieldIndex(rawData, DefaultColumn);
+ DescriptionIdx = GetFieldIndex(rawData, Description);
}
};
NKikimr::NMetadataManager::TTableRecord SerializeToRecord() const;
bool DeserializeFromRecord(const TDecoder& decoder, const Ydb::Value& r);
static NMetadata::TOperationParsingResult BuildPatchFromSettings(const NYql::TObjectSettingsImpl& settings,
const NMetadata::IOperationsManager::TModificationContext& context);
-};
-
-class TTableTiering {
-private:
- YDB_READONLY_DEF(TString, TablePath);
- YDB_READONLY(ui64, TablePathId, 0);
- YDB_READONLY_DEF(TString, Column);
- YDB_READONLY_DEF(TVector<TTieringRule>, Rules);
-public:
- void SetTablePathId(const ui64 pathId) {
- Y_VERIFY(!TablePathId);
- TablePathId = pathId;
- for (auto&& r : Rules) {
- r.SetTablePathId(pathId);
- }
- }
- NJson::TJsonValue GetDebugJson() const;
- void AddRule(TTieringRule&& tr);
-
- NOlap::TTiersInfo BuildTiersInfo() const;
+ NKikimr::NOlap::TTiersInfo BuildTiersInfo() const;
};
}
diff --git a/ydb/core/tx/tiering/snapshot.cpp b/ydb/core/tx/tiering/snapshot.cpp
index 25a1abf3463..8997364d093 100644
--- a/ydb/core/tx/tiering/snapshot.cpp
+++ b/ydb/core/tx/tiering/snapshot.cpp
@@ -20,7 +20,7 @@ bool TConfigsSnapshot::DoDeserializeFromResultSet(const Ydb::Table::ExecuteQuery
ALS_ERROR(NKikimrServices::TX_TIERING) << "cannot parse tier config from snapshot";
continue;
}
- TierConfigs.emplace(config.GetGlobalTierId(), config);
+ TierConfigs.emplace(config.GetTierName(), config);
}
}
{
@@ -36,16 +36,15 @@ bool TConfigsSnapshot::DoDeserializeFromResultSet(const Ydb::Table::ExecuteQuery
}
rulesLocal.emplace_back(std::move(tr));
}
- std::sort(rulesLocal.begin(), rulesLocal.end());
for (auto&& i : rulesLocal) {
- TableTierings[i.GetTablePath()].AddRule(std::move(i));
+ TableTierings.emplace(i.GetTieringRuleId(), std::move(i));
}
}
return true;
}
-std::optional<TTierConfig> TConfigsSnapshot::GetValue(const TGlobalTierId& key) const {
- auto it = TierConfigs.find(key);
+std::optional<TTierConfig> TConfigsSnapshot::GetTierById(const TString& tierName) const {
+ auto it = TierConfigs.find(tierName);
if (it == TierConfigs.end()) {
return {};
} else {
@@ -53,14 +52,8 @@ std::optional<TTierConfig> TConfigsSnapshot::GetValue(const TGlobalTierId& key)
}
}
-void TConfigsSnapshot::RemapTablePathToId(const TString& path, const ui64 pathId) {
- auto it = TableTierings.find(path);
- Y_VERIFY(it != TableTierings.end());
- it->second.SetTablePathId(pathId);
-}
-
-const TTableTiering* TConfigsSnapshot::GetTableTiering(const TString& tablePath) const {
- auto it = TableTierings.find(tablePath);
+const TTieringRule* TConfigsSnapshot::GetTieringById(const TString& tieringId) const {
+ auto it = TableTierings.find(tieringId);
if (it == TableTierings.end()) {
return nullptr;
} else {
@@ -68,30 +61,11 @@ const TTableTiering* TConfigsSnapshot::GetTableTiering(const TString& tablePath)
}
}
-std::vector<NKikimr::NColumnShard::NTiers::TTierConfig> TConfigsSnapshot::GetTiersForPathId(const ui64 pathId) const {
- std::vector<TTierConfig> result;
- std::set<TGlobalTierId> readyIds;
- for (auto&& i : TableTierings) {
- for (auto&& r : i.second.GetRules()) {
- if (r.GetTablePathId() == pathId) {
- auto it = TierConfigs.find(r.GetGlobalTierId());
- if (it == TierConfigs.end()) {
- ALS_ERROR(NKikimrServices::TX_TIERING) << "inconsistency tiering for " << r.GetGlobalTierId().ToString();
- continue;
- } else if (readyIds.emplace(r.GetGlobalTierId()).second) {
- result.emplace_back(it->second);
- }
- }
- }
- }
- return result;
-}
-
TString NTiers::TConfigsSnapshot::DoSerializeToString() const {
NJson::TJsonValue result = NJson::JSON_MAP;
auto& jsonTiers = result.InsertValue("tiers", NJson::JSON_MAP);
for (auto&& i : TierConfigs) {
- jsonTiers.InsertValue(i.first.ToString(), i.second.GetDebugJson());
+ jsonTiers.InsertValue(i.first, i.second.GetDebugJson());
}
auto& jsonTiering = result.InsertValue("rules", NJson::JSON_MAP);
for (auto&& i : TableTierings) {
diff --git a/ydb/core/tx/tiering/snapshot.h b/ydb/core/tx/tiering/snapshot.h
index 30f19b359d6..07d2b773706 100644
--- a/ydb/core/tx/tiering/snapshot.h
+++ b/ydb/core/tx/tiering/snapshot.h
@@ -13,18 +13,16 @@ namespace NKikimr::NColumnShard::NTiers {
class TConfigsSnapshot: public NMetadataProvider::ISnapshot {
private:
using TBase = NMetadataProvider::ISnapshot;
- using TConfigsMap = TMap<TGlobalTierId, TTierConfig>;
+ using TConfigsMap = TMap<TString, TTierConfig>;
YDB_ACCESSOR_DEF(TConfigsMap, TierConfigs);
- using TTieringMap = TMap<TString, TTableTiering>;
+ using TTieringMap = TMap<TString, TTieringRule>;
YDB_ACCESSOR_DEF(TTieringMap, TableTierings);
protected:
virtual bool DoDeserializeFromResultSet(const Ydb::Table::ExecuteQueryResult& rawData) override;
virtual TString DoSerializeToString() const override;
public:
- std::vector<TTierConfig> GetTiersForPathId(const ui64 pathId) const;
- const TTableTiering* GetTableTiering(const TString& tablePath) const;
- void RemapTablePathToId(const TString& path, const ui64 pathId);
- std::optional<TTierConfig> GetValue(const TGlobalTierId& key) const;
+ const TTieringRule* GetTieringById(const TString& tieringId) const;
+ std::optional<TTierConfig> GetTierById(const TString& tierName) const;
using TBase::TBase;
};
diff --git a/ydb/core/tx/tiering/snapshot_enrich.cpp b/ydb/core/tx/tiering/snapshot_enrich.cpp
deleted file mode 100644
index 3f7a0809f06..00000000000
--- a/ydb/core/tx/tiering/snapshot_enrich.cpp
+++ /dev/null
@@ -1,65 +0,0 @@
-#include "snapshot_enrich.h"
-#include "snapshot.h"
-
-#include <ydb/core/base/path.h>
-
-#include <util/string/join.h>
-
-namespace NKikimr::NColumnShard::NTiers {
-
-void TActorSnapshotEnrich::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
- auto current = std::dynamic_pointer_cast<TConfigsSnapshot>(Original);
- Y_VERIFY(current);
- auto g = PassAwayGuard();
-
- auto* info = ev->Get();
- const auto& request = info->Request;
-
- if (request->ResultSet.empty()) {
- Controller->EnrichProblem("cannot resolve table path to ids");
- return;
- }
-
- for (auto&& i : request->ResultSet) {
- const TString path = "/" + JoinSeq("/", i.Path);
- switch (i.Status) {
- case TNavigate::EStatus::Ok:
- Cache->MutableRemapper().emplace(path, i.TableId.PathId.LocalPathId);
- current->RemapTablePathToId(path, i.TableId.PathId.LocalPathId);
- break;
- default:
- Controller->EnrichProblem(::ToString(i.Status) + " for '" + path + "'");
- return;
- }
- }
- for (auto&& i : current->GetTableTierings()) {
- Y_VERIFY(i.second.GetTablePathId());
- }
- Controller->Enriched(Original);
-}
-
-void TActorSnapshotEnrich::Bootstrap() {
- Become(&TThis::StateMain);
- auto current = std::dynamic_pointer_cast<TConfigsSnapshot>(Original);
- Y_VERIFY(current);
- const auto& remapper = Cache->GetRemapper();
- auto request = MakeHolder<NSchemeCache::TSchemeCacheNavigate>();
- request->DatabaseName = NKikimr::CanonizePath(AppData()->TenantName);
- for (auto&& i : current->GetTableTierings()) {
- auto it = remapper.find(i.second.GetTablePath());
- if (it != remapper.end()) {
- current->RemapTablePathToId(it->first, it->second);
- } else {
- auto& entry = request->ResultSet.emplace_back();
- entry.Operation = TNavigate::OpTable;
- entry.Path = NKikimr::SplitPath(i.second.GetTablePath());
- }
- }
- if (request->ResultSet.empty()) {
- Controller->Enriched(Original);
- } else {
- Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()));
- }
-}
-
-}
diff --git a/ydb/core/tx/tiering/snapshot_enrich.h b/ydb/core/tx/tiering/snapshot_enrich.h
deleted file mode 100644
index 2a24ba2226a..00000000000
--- a/ydb/core/tx/tiering/snapshot_enrich.h
+++ /dev/null
@@ -1,53 +0,0 @@
-#pragma once
-
-#include <ydb/core/protos/flat_scheme_op.pb.h>
-#include <ydb/core/tx/scheme_cache/scheme_cache.h>
-#include <ydb/services/metadata/service.h>
-
-#include <library/cpp/json/writer/json_value.h>
-
-namespace NKikimr::NColumnShard::NTiers {
-
-class TTablesDecoderCache {
-private:
- using TRemapper = TMap<TString, ui64>;
- YDB_ACCESSOR_DEF(TRemapper, Remapper);
-public:
- using TPtr = std::shared_ptr<TTablesDecoderCache>;
-};
-
-class TActorSnapshotEnrich: public TActorBootstrapped<TActorSnapshotEnrich> {
-private:
- using TBase = TActorBootstrapped<TActorSnapshotEnrich>;
- using TNavigate = NSchemeCache::TSchemeCacheNavigate;
- NMetadataProvider::ISnapshot::TPtr Original;
- NMetadataProvider::ISnapshotAcceptorController::TPtr Controller;
- TTablesDecoderCache::TPtr Cache;
-private:
- void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev);
-public:
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::TIERING_SNAPSHOT_ENRICH;
- }
-
- STATEFN(StateMain) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle);
- default:
- break;
- }
- }
-
- TActorSnapshotEnrich(NMetadataProvider::ISnapshot::TPtr original,
- NMetadataProvider::ISnapshotAcceptorController::TPtr controller, TTablesDecoderCache::TPtr cache)
- : Original(original)
- , Controller(controller)
- , Cache(cache)
- {
-
- }
-
- void Bootstrap();
-};
-
-}
diff --git a/ydb/core/tx/tiering/tier/CMakeLists.txt b/ydb/core/tx/tiering/tier/CMakeLists.txt
index 523f34fb8c2..8ab14cb66a5 100644
--- a/ydb/core/tx/tiering/tier/CMakeLists.txt
+++ b/ydb/core/tx/tiering/tier/CMakeLists.txt
@@ -13,6 +13,7 @@ target_link_libraries(tx-tiering-tier PUBLIC
yutil
services-metadata-initializer
services-metadata-abstract
+ services-metadata-secret
)
target_sources(tx-tiering-tier PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/tier/object.cpp
@@ -25,6 +26,7 @@ target_link_libraries(tx-tiering-tier.global PUBLIC
yutil
services-metadata-initializer
services-metadata-abstract
+ services-metadata-secret
)
target_sources(tx-tiering-tier.global PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/tier/manager.cpp
diff --git a/ydb/core/tx/tiering/tier/initializer.cpp b/ydb/core/tx/tiering/tier/initializer.cpp
index 4567b99fef6..d414499460d 100644
--- a/ydb/core/tx/tiering/tier/initializer.cpp
+++ b/ydb/core/tx/tiering/tier/initializer.cpp
@@ -12,11 +12,6 @@ TVector<NKikimr::NMetadataInitializer::ITableModifier::TPtr> TTiersInitializer::
request.add_primary_key("tierName");
{
auto& column = *request.add_columns();
- column.set_name("ownerPath");
- column.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::STRING);
- }
- {
- auto& column = *request.add_columns();
column.set_name("tierName");
column.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::STRING);
}
diff --git a/ydb/core/tx/tiering/tier/object.cpp b/ydb/core/tx/tiering/tier/object.cpp
index 638d2614f7a..c09d02ced04 100644
--- a/ydb/core/tx/tiering/tier/object.cpp
+++ b/ydb/core/tx/tiering/tier/object.cpp
@@ -1,5 +1,7 @@
#include "object.h"
+
#include <ydb/services/metadata/manager/ydb_value_operator.h>
+#include <ydb/services/metadata/secret/fetcher.h>
#include <library/cpp/json/writer/json_value.h>
#include <library/cpp/protobuf/json/proto2json.h>
@@ -8,7 +10,6 @@ namespace NKikimr::NColumnShard::NTiers {
NJson::TJsonValue TTierConfig::GetDebugJson() const {
NJson::TJsonValue result = NJson::JSON_MAP;
- result.InsertValue(TDecoder::OwnerPath, OwnerPath);
result.InsertValue(TDecoder::TierName, TierName);
NProtobufJson::Proto2Json(ProtoConfig, result.InsertValue(TDecoder::TierConfig, NJson::JSON_MAP));
return result;
@@ -19,10 +20,6 @@ bool TTierConfig::IsSame(const TTierConfig& item) const {
}
bool TTierConfig::DeserializeFromRecord(const TDecoder& decoder, const Ydb::Value& r) {
- if (!decoder.Read(decoder.GetOwnerPathIdx(), OwnerPath, r)) {
- return false;
- }
- OwnerPath = TFsPath(OwnerPath).Fix().GetPath();
if (!decoder.Read(decoder.GetTierNameIdx(), TierName, r)) {
return false;
}
@@ -34,7 +31,6 @@ bool TTierConfig::DeserializeFromRecord(const TDecoder& decoder, const Ydb::Valu
NKikimr::NMetadataManager::TTableRecord TTierConfig::SerializeToRecord() const {
NMetadataManager::TTableRecord result;
- result.SetColumn(TDecoder::OwnerPath, NMetadataManager::TYDBValue::Bytes(OwnerPath));
result.SetColumn(TDecoder::TierName, NMetadataManager::TYDBValue::Bytes(TierName));
result.SetColumn(TDecoder::TierConfig, NMetadataManager::TYDBValue::Bytes(ProtoConfig.DebugString()));
return result;
@@ -55,12 +51,6 @@ NMetadata::TOperationParsingResult TTierConfig::BuildPatchFromSettings(const NYq
NKikimr::NMetadataManager::TTableRecord result;
result.SetColumn(TDecoder::TierName, NMetadataManager::TYDBValue::Bytes(settings.GetObjectId()));
{
- auto it = settings.GetFeatures().find(TDecoder::OwnerPath);
- if (it != settings.GetFeatures().end()) {
- result.SetColumn(TDecoder::OwnerPath, NMetadataManager::TYDBValue::Bytes(it->second));
- }
- }
- {
auto it = settings.GetFeatures().find(TDecoder::TierConfig);
if (it != settings.GetFeatures().end()) {
TTierProto proto;
@@ -74,6 +64,17 @@ NMetadata::TOperationParsingResult TTierConfig::BuildPatchFromSettings(const NYq
return result;
}
+NKikimrSchemeOp::TS3Settings TTierConfig::GetPatchedConfig(
+ std::shared_ptr<NMetadata::NSecret::TSnapshot> secrets) const
+{
+ auto config = ProtoConfig.GetObjectStorage();
+ if (secrets) {
+ secrets->PatchString(*config.MutableAccessKey());
+ secrets->PatchString(*config.MutableSecretKey());
+ }
+ return config;
+}
+
std::vector<Ydb::Column> TTierConfig::TDecoder::GetPKColumns() {
return { NMetadataManager::TYDBColumn::Bytes(TierName) };
}
@@ -81,7 +82,6 @@ std::vector<Ydb::Column> TTierConfig::TDecoder::GetPKColumns() {
std::vector<Ydb::Column> TTierConfig::TDecoder::GetColumns() {
return
{
- NMetadataManager::TYDBColumn::Bytes(OwnerPath),
NMetadataManager::TYDBColumn::Bytes(TierName),
NMetadataManager::TYDBColumn::Bytes(TierConfig)
};
diff --git a/ydb/core/tx/tiering/tier/object.h b/ydb/core/tx/tiering/tier/object.h
index 3d0aef8392b..bbe3ca1071a 100644
--- a/ydb/core/tx/tiering/tier/object.h
+++ b/ydb/core/tx/tiering/tier/object.h
@@ -1,6 +1,5 @@
#pragma once
#include <ydb/core/protos/flat_scheme_op.pb.h>
-#include <ydb/core/tx/tiering/common/global_tier_id.h>
#include <ydb/services/metadata/abstract/decoder.h>
#include <ydb/services/metadata/manager/preparation_controller.h>
#include <ydb/services/metadata/manager/table_record.h>
@@ -9,38 +8,39 @@
#include <library/cpp/json/writer/json_value.h>
+namespace NKikimr::NMetadata::NSecret {
+class TSnapshot;
+}
+
namespace NKikimr::NColumnShard::NTiers {
class TTierConfig: public NMetadataManager::TObject<TTierConfig> {
private:
using TTierProto = NKikimrSchemeOp::TStorageTierConfig;
- YDB_ACCESSOR_DEF(TString, OwnerPath);
YDB_ACCESSOR_DEF(TString, TierName);
YDB_ACCESSOR_DEF(TTierProto, ProtoConfig);
public:
TTierConfig() = default;
- TTierConfig(const TString& ownerPath, const TString& tierName)
- : OwnerPath(ownerPath)
- , TierName(tierName)
+ TTierConfig(const TString& tierName)
+ : TierName(tierName)
{
}
+ NKikimrSchemeOp::TS3Settings GetPatchedConfig(std::shared_ptr<NMetadata::NSecret::TSnapshot> secrets) const;
+
class TDecoder: public NInternal::TDecoderBase {
private:
- YDB_READONLY(i32, OwnerPathIdx, -1);
YDB_READONLY(i32, TierNameIdx, -1);
YDB_READONLY(i32, TierConfigIdx, -1);
public:
- static inline const TString OwnerPath = "ownerPath";
static inline const TString TierName = "tierName";
static inline const TString TierConfig = "tierConfig";
static std::vector<Ydb::Column> GetPKColumns();
static std::vector<Ydb::Column> GetColumns();
static std::vector<TString> GetPKColumnIds();
TDecoder(const Ydb::ResultSet& rawData) {
- OwnerPathIdx = GetFieldIndex(rawData, OwnerPath);
TierNameIdx = GetFieldIndex(rawData, TierName);
TierConfigIdx = GetFieldIndex(rawData, TierConfig);
}
@@ -55,10 +55,6 @@ public:
NMetadataManager::IAlterPreparationController<TTierConfig>::TPtr controller,
const NMetadata::IOperationsManager::TModificationContext& context);
- TGlobalTierId GetGlobalTierId() const {
- return TGlobalTierId(OwnerPath, TierName);
- }
-
bool NeedExport() const {
return ProtoConfig.HasObjectStorage();
}
diff --git a/ydb/core/tx/tiering/ut/ut_tiers.cpp b/ydb/core/tx/tiering/ut/ut_tiers.cpp
index 3c657d30766..8b950972def 100644
--- a/ydb/core/tx/tiering/ut/ut_tiers.cpp
+++ b/ydb/core/tx/tiering/ut/ut_tiers.cpp
@@ -65,7 +65,7 @@ public:
ColumnShardCount: %d
TtlSettings: {
Tiering: {
- EnableTiering: true
+ UseTiering: "tiering1"
}
}
Sharding {
@@ -85,6 +85,28 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) {
const TString ConfigProtoStr1 = "Name : \"abc1\"";
const TString ConfigProtoStr2 = "Name : \"abc2\"";
+ const TString ConfigTiering1Str = R"({
+ "rules" : [
+ {
+ "tierName" : "tier1",
+ "durationForEvict" : "10d"
+ },
+ {
+ "tierName" : "tier2",
+ "durationForEvict" : "20d"
+ }
+ ]
+ })";
+
+ const TString ConfigTiering2Str = R"({
+ "rules" : [
+ {
+ "tierName" : "tier1",
+ "durationForEvict" : "10d"
+ }
+ ]
+ })";
+
class TJsonChecker {
private:
YDB_ACCESSOR_DEF(TString, Path);
@@ -121,7 +143,8 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) {
YDB_READONLY_FLAG(Found, false);
YDB_ACCESSOR(ui32, ExpectedTieringsCount, 1);
YDB_ACCESSOR(ui32, ExpectedTiersCount, 1);
- using TKeyCheckers = TMap<NTiers::TGlobalTierId, TJsonChecker>;
+
+ using TKeyCheckers = TMap<TString, TJsonChecker>;
YDB_ACCESSOR_DEF(TKeyCheckers, Checkers);
public:
void ResetConditions() {
@@ -161,42 +184,31 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) {
void CheckFound(NMetadataProvider::TEvRefreshSubscriberData* event) {
auto snapshot = event->GetSnapshotAs<NTiers::TConfigsSnapshot>();
- Y_VERIFY(!!snapshot);
- auto* tInfo = snapshot->GetTableTiering("/Root/olapStore/olapTable");
- if (ExpectedTieringsCount) {
- if (!tInfo) {
- Cerr << "tiering not found: " << snapshot->SerializeToString() << Endl;
- return ;
- }
- if (tInfo->GetRules().size() != ExpectedTieringsCount) {
- Cerr << "TieringsCount incorrect: " << snapshot->SerializeToString() << Endl;
- return;
- }
- } else if (tInfo) {
- Cerr << "tiering found but its incorrect: " << snapshot->SerializeToString() << Endl;
+ if (!snapshot) {
return;
}
- for (auto&& [_, tiering] : snapshot->GetTableTierings()) {
- if (tiering.GetTablePathId() == 0) {
- Cerr << "PathId not initialized: " << snapshot->SerializeToString() << Endl;
- return;
- } else if (tiering.GetTablePathId() == InvalidLocalPathId) {
- Cerr << "PathId invalid: " << snapshot->SerializeToString() << Endl;
- return;
- } else {
- Cerr << "PathId: " << tiering.GetTablePathId() << Endl;
- }
+ const auto& tierings = snapshot->GetTableTierings();
+ if (tierings.size() != ExpectedTieringsCount) {
+ Cerr << "TieringsCount incorrect: " << snapshot->SerializeToString() << ";expectation=" << ExpectedTieringsCount << Endl;
+ return;
}
if (ExpectedTiersCount != snapshot->GetTierConfigs().size()) {
- Cerr << "TieringsCount incorrect: " << snapshot->SerializeToString() << Endl;
+ Cerr << "TiersCount incorrect: " << snapshot->SerializeToString() << ";expectation=" << ExpectedTiersCount << Endl;
return;
}
for (auto&& i : Checkers) {
- auto value = snapshot->GetValue(i.first);
NJson::TJsonValue jsonData;
- NProtobufJson::Proto2Json(value->GetProtoConfig(), jsonData);
+ if (i.first.StartsWith("TIER.")) {
+ auto value = snapshot->GetTierById(i.first.substr(5));
+ NProtobufJson::Proto2Json(value->GetProtoConfig(), jsonData);
+ } else if (i.first.StartsWith("TIERING_RULE.")) {
+ auto value = snapshot->GetTierById(i.first.substr(13));
+ NProtobufJson::Proto2Json(value->GetProtoConfig(), jsonData);
+ } else {
+ Y_VERIFY(false);
+ }
if (!i.second.Check(jsonData)) {
- Cerr << "config value incorrect:" << snapshot->SerializeToString() << ";snapshot_check_path=" << i.first.ToString() << Endl;
+ Cerr << "config value incorrect:" << snapshot->SerializeToString() << ";snapshot_check_path=" << i.first << Endl;
Cerr << "json path incorrect:" << jsonData << ";" << i.second.GetDebugString() << Endl;
return;
}
@@ -259,16 +271,16 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) {
lHelper.CreateTestOlapTable();
{
TTestCSEmulator* emulator = new TTestCSEmulator;
- emulator->MutableCheckers().emplace(NTiers::TGlobalTierId("/Root/olapStore", "tier1"), TJsonChecker("Name", "abc"));
+ emulator->MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "abc"));
runtime.Register(emulator);
runtime.SimulateSleep(TDuration::Seconds(10));
Cerr << "Initialization finished" << Endl;
lHelper.StartSchemaRequest("CREATE OBJECT tier1 ( "
- "TYPE TIER) WITH (tierConfig = `" + ConfigProtoStr + "`, ownerPath = `/Root/olapStore`)");
+ "TYPE TIER) WITH (tierConfig = `" + ConfigProtoStr + "`)");
lHelper.StartSchemaRequest("CREATE OBJECT tier1 ("
- "TYPE TIERING_RULE) WITH (tierName = tier1, tablePath = `/Root/olapStore/olapTable`, "
- "ownerPath = `/Root/olapStore`, column = timestamp, durationForEvict = `10d` )");
+ "TYPE TIERING_RULE) WITH (tierName = tier1, "
+ "defaultColumn = timestamp, description = `" + ConfigTiering1Str + "` )");
{
const TInstant start = Now();
while (!emulator->IsFound() && Now() - start < TDuration::Seconds(2000)) {
@@ -281,7 +293,7 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) {
"TYPE TIER) SET tierConfig = `" + ConfigProtoStr1 + "`");
emulator->ResetConditions();
- emulator->MutableCheckers().emplace(NTiers::TGlobalTierId("/Root/olapStore", "tier1"), TJsonChecker("Name", "abc1"));
+ emulator->MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "abc1"));
{
const TInstant start = Now();
while (!emulator->IsFound() && Now() - start < TDuration::Seconds(2000)) {
@@ -349,34 +361,34 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) {
Cerr << "Initialization finished" << Endl;
lHelper.StartSchemaRequest("CREATE OBJECT tier1 ( "
- "TYPE TIER) WITH (tierConfig = `" + ConfigProtoStr1 + "`, ownerPath = `/Root/olapStore`)");
- lHelper.StartSchemaRequest("CREATE OBJECT tier1 ("
+ "TYPE TIER) WITH (tierConfig = `" + ConfigProtoStr1 + "`)");
+ lHelper.StartSchemaRequest("CREATE OBJECT tiering1 ("
"TYPE TIERING_RULE) "
- "WITH (ownerPath = `/Root/olapStore`, tierName = tier1, tablePath = `/Root/olapStore/olapTable`, column = timestamp, durationForEvict = `10d` "
+ "WITH (tierName = tier1, defaultColumn = timestamp, description = `" + ConfigTiering1Str + "` "
")");
{
TTestCSEmulator emulator;
- emulator.MutableCheckers().emplace(NTiers::TGlobalTierId("/Root/olapStore", "tier1"), TJsonChecker("Name", "abc1"));
+ emulator.MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "abc1"));
emulator.SetExpectedTieringsCount(1);
emulator.SetExpectedTiersCount(1);
emulator.CheckRuntime(runtime);
}
lHelper.StartSchemaRequest("CREATE OBJECT tier2 ( "
- "TYPE TIER) WITH (ownerPath = `/Root/olapStore`, tierConfig = `" + ConfigProtoStr2 + "`)");
- lHelper.StartSchemaRequest("CREATE OBJECT tier2 ("
- "TYPE TIERING_RULE) WITH (tierName = tier2, tablePath = `/Root/olapStore/olapTable`, ownerPath = `/Root/olapStore`, column = timestamp, durationForEvict = `20d` )");
+ "TYPE TIER) WITH (tierConfig = `" + ConfigProtoStr2 + "`)");
+ lHelper.StartSchemaRequest("CREATE OBJECT tiering2 ("
+ "TYPE TIERING_RULE) WITH (tierName = tier2, defaultColumn = timestamp, description = `" + ConfigTiering2Str + "` )");
{
TTestCSEmulator emulator;
- emulator.MutableCheckers().emplace(NTiers::TGlobalTierId("/Root/olapStore", "tier1"), TJsonChecker("Name", "abc1"));
- emulator.MutableCheckers().emplace(NTiers::TGlobalTierId("/Root/olapStore", "tier2"), TJsonChecker("Name", "abc2"));
+ emulator.MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "abc1"));
+ emulator.MutableCheckers().emplace("TIER.tier2", TJsonChecker("Name", "abc2"));
emulator.SetExpectedTieringsCount(2);
emulator.SetExpectedTiersCount(2);
emulator.CheckRuntime(runtime);
}
lHelper.StartSchemaRequest("DROP OBJECT tier2 (TYPE TIER)");
- lHelper.StartSchemaRequest("DROP OBJECT tier2 (TYPE TIERING_RULE)");
+ lHelper.StartSchemaRequest("DROP OBJECT tiering2 (TYPE TIERING_RULE)");
{
TTestCSEmulator emulator;
emulator.SetExpectedTieringsCount(1);
@@ -384,7 +396,7 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) {
emulator.CheckRuntime(runtime);
}
lHelper.StartSchemaRequest("DROP OBJECT tier1 (TYPE TIER)");
- lHelper.StartSchemaRequest("DROP OBJECT tier1 (TYPE TIERING_RULE)");
+ lHelper.StartSchemaRequest("DROP OBJECT tiering1 (TYPE TIERING_RULE)");
{
TTestCSEmulator emulator;
emulator.SetExpectedTieringsCount(0);
@@ -405,8 +417,8 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) {
VerifySSL: false
Endpoint: "storage.cloud-preprod.yandex.net"
Bucket: "tiering-test-01"
- AccessKey: "..."
- SecretKey: "..."
+ AccessKey: "USId:root@builtin:secretAccessKey"
+ SecretKey: "USId:root@builtin:secretSecretKey"
ProxyHost: "localhost"
ProxyPort: 8080
ProxyScheme: HTTP
@@ -420,6 +432,8 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) {
ObjectStorage : {
Endpoint: "fake"
Bucket: "fake"
+ AccessKey: "USId:root@builtin:secretAccessKey"
+ SecretKey: "USId:root@builtin:secretSecretKey"
}
)";
const TString TierEndpoint = "fake";
@@ -451,36 +465,43 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) {
auto sender = runtime.AllocateEdgeActor();
server->SetupRootStoragePools(sender);
- TLocalHelper lHelper(*server);
- lHelper.CreateTestOlapTable("olapTable");
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_NOTICE);
runtime.SetLogPriority(NKikimrServices::TX_COLUMNSHARD, NLog::PRI_DEBUG);
runtime.SetLogPriority(NKikimrServices::BG_TASKS, NLog::PRI_DEBUG);
// runtime.SetLogPriority(NKikimrServices::TX_PROXY_SCHEME_CACHE, NLog::PRI_DEBUG);
- Cerr << "Wait initialization" << Endl;
- runtime.SimulateSleep(TDuration::Seconds(20));
- Cerr << "Initialization finished" << Endl;
- lHelper.StartSchemaRequest("CREATE OBJECT fakeTier1 ( "
- "TYPE TIER) WITH (tierConfig = `" + TierConfigProtoStr + "`, ownerPath = `/Root/olapStore`)");
- lHelper.StartSchemaRequest("CREATE OBJECT fakeTier1 ("
- "TYPE TIERING_RULE) WITH (tierName = fakeTier1, ownerPath = `/Root/olapStore`, tablePath = `/Root/olapStore/olapTable`, column = timestamp, durationForEvict = `10d` )");
+ TLocalHelper lHelper(*server);
+ lHelper.StartSchemaRequest("CREATE OBJECT secretAccessKey ( "
+ "TYPE SECRET) WITH (value = ak)");
+ lHelper.StartSchemaRequest("CREATE OBJECT secretSecretKey ( "
+ "TYPE SECRET) WITH (value = sk)");
+ Singleton<NKikimr::NWrappers::NExternalStorage::TFakeExternalStorage>()->SetSecretKey("sk");
+
+ lHelper.StartSchemaRequest("CREATE OBJECT tier1 ( "
+ "TYPE TIER) WITH (tierConfig = `" + TierConfigProtoStr + "`)");
+ lHelper.StartSchemaRequest("CREATE OBJECT tiering1 ("
+ "TYPE TIERING_RULE) WITH (tierName = tiering1, defaultColumn = timestamp, description = `" + ConfigTiering1Str + "` )");
- lHelper.StartSchemaRequest("CREATE OBJECT fakeTier2 ( "
- "TYPE TIER) WITH (tierConfig = `" + TierConfigProtoStr + "`, ownerPath = `/Root/olapStore`)");
- lHelper.StartSchemaRequest("CREATE OBJECT fakeTier2 ("
- "TYPE TIERING_RULE) WITH (tierName = fakeTier2, ownerPath = `/Root/olapStore`, tablePath = `/Root/olapStore/olapTable`, column = timestamp, durationForEvict = `20d` )");
+ lHelper.StartSchemaRequest("CREATE OBJECT tier2 ( "
+ "TYPE TIER) WITH (tierConfig = `" + TierConfigProtoStr + "`)");
+ lHelper.StartSchemaRequest("CREATE OBJECT tiering2 ("
+ "TYPE TIERING_RULE) WITH (tierName = tiering2, defaultColumn = timestamp, description = `" + ConfigTiering2Str + "` )");
{
- TTestCSEmulator emulator;
- emulator.MutableCheckers().emplace(NTiers::TGlobalTierId("/Root/olapStore", "fakeTier1"), TJsonChecker("Name", "fakeTier"));
- emulator.MutableCheckers().emplace(NTiers::TGlobalTierId("/Root/olapStore", "fakeTier2"), TJsonChecker("ObjectStorage.Endpoint", TierEndpoint));
- emulator.SetExpectedTieringsCount(2);
- emulator.SetExpectedTiersCount(2);
- emulator.CheckRuntime(runtime);
+ TTestCSEmulator* emulator = new TTestCSEmulator;
+ runtime.Register(emulator);
+ emulator->MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "fakeTier"));
+ emulator->MutableCheckers().emplace("TIER.tier2", TJsonChecker("ObjectStorage.Endpoint", TierEndpoint));
+ emulator->SetExpectedTieringsCount(2);
+ emulator->SetExpectedTiersCount(2);
+ emulator->CheckRuntime(runtime);
}
+ lHelper.CreateTestOlapTable("olapTable");
+ Cerr << "Wait tables" << Endl;
+ runtime.SimulateSleep(TDuration::Seconds(20));
+ Cerr << "Initialization tables" << Endl;
Cerr << "Insert..." << Endl;
- const TInstant pkStart = Now() - TDuration::Days(15);
+ const TInstant pkStart = Now() - TDuration::Days(5);
ui32 idx = 0;
lHelper.SendDataViaActorSystem("/Root/olapStore/olapTable", 0, (pkStart + TDuration::Seconds(2 * idx++)).GetValue(), 2000);
{
diff --git a/ydb/core/wrappers/abstract.cpp b/ydb/core/wrappers/abstract.cpp
index 13c4f1bdddc..c13430b7e87 100644
--- a/ydb/core/wrappers/abstract.cpp
+++ b/ydb/core/wrappers/abstract.cpp
@@ -12,7 +12,7 @@ IExternalStorageOperator::TPtr IExternalStorageConfig::ConstructStorageOperator(
IExternalStorageConfig::TPtr IExternalStorageConfig::Construct(const NKikimrSchemeOp::TS3Settings& settings) {
if (settings.GetEndpoint() == "fake") {
- return std::make_shared<TFakeExternalStorageConfig>(settings.GetBucket());
+ return std::make_shared<TFakeExternalStorageConfig>(settings.GetBucket(), settings.GetSecretKey());
} else {
return std::make_shared <TS3ExternalStorageConfig>(settings);
}
diff --git a/ydb/core/wrappers/fake_storage.h b/ydb/core/wrappers/fake_storage.h
index 09c23ee7a2a..cc74a36c86f 100644
--- a/ydb/core/wrappers/fake_storage.h
+++ b/ydb/core/wrappers/fake_storage.h
@@ -46,6 +46,8 @@ public:
class TFakeExternalStorage {
private:
+ YDB_ACCESSOR_DEF(TString, SecretKey);
+
mutable TMutex Mutex;
mutable TMap<TString, TFakeBucketStorage> BucketStorages;
TEvListObjectsResponse::TResult BuildListObjectsResult(const TEvListObjectsRequest::TRequest& request) const;
@@ -103,55 +105,54 @@ public:
class TFakeExternalStorageOperator: public IExternalStorageOperator {
private:
const TString Bucket;
+ const TString SecretKey;
+
+ template <class TEvent>
+ void ExecuteImpl(TEvent& ev) const {
+ ev->Get()->MutableRequest().WithBucket(Bucket);
+ Y_VERIFY(SecretKey == Singleton<TFakeExternalStorage>()->GetSecretKey());
+ Singleton<TFakeExternalStorage>()->Execute(ev);
+ }
+
public:
- TFakeExternalStorageOperator(const TString& bucket)
+ TFakeExternalStorageOperator(const TString& bucket, const TString& secretKey)
: Bucket(bucket)
+ , SecretKey(secretKey)
{
}
virtual void Execute(TEvCheckObjectExistsRequest::TPtr& ev) const override {
- ev->Get()->MutableRequest().WithBucket(Bucket);
- Singleton<TFakeExternalStorage>()->Execute(ev);
+ ExecuteImpl(ev);
}
virtual void Execute(TEvListObjectsRequest::TPtr& ev) const override {
- ev->Get()->MutableRequest().WithBucket(Bucket);
- Singleton<TFakeExternalStorage>()->Execute(ev);
+ ExecuteImpl(ev);
}
virtual void Execute(TEvGetObjectRequest::TPtr& ev) const override {
- ev->Get()->MutableRequest().WithBucket(Bucket);
- Singleton<TFakeExternalStorage>()->Execute(ev);
+ ExecuteImpl(ev);
}
virtual void Execute(TEvHeadObjectRequest::TPtr& ev) const override {
- ev->Get()->MutableRequest().WithBucket(Bucket);
- Singleton<TFakeExternalStorage>()->Execute(ev);
+ ExecuteImpl(ev);
}
virtual void Execute(TEvPutObjectRequest::TPtr& ev) const override {
- ev->Get()->MutableRequest().WithBucket(Bucket);
- Singleton<TFakeExternalStorage>()->Execute(ev);
+ ExecuteImpl(ev);
}
virtual void Execute(TEvDeleteObjectRequest::TPtr& ev) const override {
- ev->Get()->MutableRequest().WithBucket(Bucket);
- Singleton<TFakeExternalStorage>()->Execute(ev);
+ ExecuteImpl(ev);
}
virtual void Execute(TEvDeleteObjectsRequest::TPtr& ev) const override {
- ev->Get()->MutableRequest().WithBucket(Bucket);
- Singleton<TFakeExternalStorage>()->Execute(ev);
+ ExecuteImpl(ev);
}
virtual void Execute(TEvCreateMultipartUploadRequest::TPtr& ev) const override {
- ev->Get()->MutableRequest().WithBucket(Bucket);
- Singleton<TFakeExternalStorage>()->Execute(ev);
+ ExecuteImpl(ev);
}
virtual void Execute(TEvUploadPartRequest::TPtr& ev) const override {
- ev->Get()->MutableRequest().WithBucket(Bucket);
- Singleton<TFakeExternalStorage>()->Execute(ev);
+ ExecuteImpl(ev);
}
virtual void Execute(TEvCompleteMultipartUploadRequest::TPtr& ev) const override {
- ev->Get()->MutableRequest().WithBucket(Bucket);
- Singleton<TFakeExternalStorage>()->Execute(ev);
+ ExecuteImpl(ev);
}
virtual void Execute(TEvAbortMultipartUploadRequest::TPtr& ev) const override {
- ev->Get()->MutableRequest().WithBucket(Bucket);
- Singleton<TFakeExternalStorage>()->Execute(ev);
+ ExecuteImpl(ev);
}
};
} // NKikimr::NWrappers::NExternalStorage
diff --git a/ydb/core/wrappers/fake_storage_config.cpp b/ydb/core/wrappers/fake_storage_config.cpp
index 0d2f9650f2a..5cfd33e0294 100644
--- a/ydb/core/wrappers/fake_storage_config.cpp
+++ b/ydb/core/wrappers/fake_storage_config.cpp
@@ -10,7 +10,7 @@ TString TFakeExternalStorageConfig::DoGetStorageId() const {
}
IExternalStorageOperator::TPtr TFakeExternalStorageConfig::DoConstructStorageOperator() const {
- return std::make_shared<TFakeExternalStorageOperator>(Bucket);
+ return std::make_shared<TFakeExternalStorageOperator>(Bucket, SecretKey);
}
}
diff --git a/ydb/core/wrappers/fake_storage_config.h b/ydb/core/wrappers/fake_storage_config.h
index e6ce528a7c5..1b3154de18b 100644
--- a/ydb/core/wrappers/fake_storage_config.h
+++ b/ydb/core/wrappers/fake_storage_config.h
@@ -14,12 +14,14 @@ namespace NKikimr::NWrappers::NExternalStorage {
class TFakeExternalStorageConfig: public IExternalStorageConfig {
private:
const TString Bucket;
+ const TString SecretKey;
protected:
virtual TString DoGetStorageId() const override;
virtual IExternalStorageOperator::TPtr DoConstructStorageOperator() const override;
public:
- TFakeExternalStorageConfig(const TString& bucket)
+ TFakeExternalStorageConfig(const TString& bucket, const TString& secretKey)
: Bucket(bucket)
+ , SecretKey(secretKey)
{
}
diff --git a/ydb/services/metadata/abstract/common.h b/ydb/services/metadata/abstract/common.h
index f3cabd77350..3b71104341a 100644
--- a/ydb/services/metadata/abstract/common.h
+++ b/ydb/services/metadata/abstract/common.h
@@ -44,6 +44,11 @@ public:
}
template <class TSnapshot>
+ std::shared_ptr<TSnapshot> GetSnapshotPtrAs() const {
+ return std::dynamic_pointer_cast<TSnapshot>(Snapshot);
+ }
+
+ template <class TSnapshot>
std::shared_ptr<TSnapshot> GetValidatedSnapshotAs() const {
auto result = dynamic_pointer_cast<TSnapshot>(Snapshot);
Y_VERIFY(result);
diff --git a/ydb/services/metadata/secret/secret.cpp b/ydb/services/metadata/secret/secret.cpp
index ccd45b426e2..353fbb3f8e4 100644
--- a/ydb/services/metadata/secret/secret.cpp
+++ b/ydb/services/metadata/secret/secret.cpp
@@ -87,4 +87,27 @@ std::vector<TString> TSecret::TDecoder::GetPKColumnIds() {
return { OwnerUserId, SecretId };
}
+TString TSecretId::SerializeToString() const {
+ TStringBuilder sb;
+ sb << "USId:" << OwnerUserId << ":" << SecretId;
+ return sb;
+}
+
+bool TSecretId::DeserializeFromString(const TString& info) {
+ static const TString prefix = "USId:";
+ if (!info.StartsWith(prefix)) {
+ return false;
+ }
+ TStringBuf sb(info.data(), info.size());
+ sb.Skip(prefix.size());
+ TStringBuf uId;
+ TStringBuf sId;
+ if (!sb.TrySplit(':', uId, sId)) {
+ return false;
+ }
+ OwnerUserId = uId;
+ SecretId = sId;
+ return true;
+}
+
}
diff --git a/ydb/services/metadata/secret/secret.h b/ydb/services/metadata/secret/secret.h
index ca6b653f323..0c7c8b833ce 100644
--- a/ydb/services/metadata/secret/secret.h
+++ b/ydb/services/metadata/secret/secret.h
@@ -21,6 +21,9 @@ public:
{
}
+ bool DeserializeFromString(const TString& info);
+ TString SerializeToString() const;
+
bool operator<(const TSecretId& item) const {
return std::tie(OwnerUserId, SecretId) < std::tie(item.OwnerUserId, item.SecretId);
}
diff --git a/ydb/services/metadata/secret/snapshot.cpp b/ydb/services/metadata/secret/snapshot.cpp
index f97ba3a3bf4..89568d5ab5a 100644
--- a/ydb/services/metadata/secret/snapshot.cpp
+++ b/ydb/services/metadata/secret/snapshot.cpp
@@ -44,4 +44,17 @@ TString TSnapshot::DoSerializeToString() const {
return sb;
}
+bool TSnapshot::PatchString(TString& stringForPath) const {
+ TSecretId sId;
+ if (!sId.DeserializeFromString(stringForPath)) {
+ return false;
+ }
+ auto it = Secrets.find(sId);
+ if (it == Secrets.end()) {
+ return false;
+ }
+ stringForPath = it->second.GetValue();
+ return true;
+}
+
}
diff --git a/ydb/services/metadata/secret/snapshot.h b/ydb/services/metadata/secret/snapshot.h
index c25ea23600f..541dc985aef 100644
--- a/ydb/services/metadata/secret/snapshot.h
+++ b/ydb/services/metadata/secret/snapshot.h
@@ -18,6 +18,8 @@ protected:
virtual TString DoSerializeToString() const override;
public:
using TBase::TBase;
+
+ bool PatchString(TString& stringForPath) const;
};
}