diff options
author | ilnaz <ilnaz@ydb.tech> | 2023-05-10 11:06:13 +0300 |
---|---|---|
committer | ilnaz <ilnaz@ydb.tech> | 2023-05-10 11:06:13 +0300 |
commit | 832415b6bf33a23a003195441f74d66164038084 (patch) | |
tree | 137583135bb06939e4586b4846bf7eaa162a0278 | |
parent | 4d6feceac9ea8999f879e1fa5539da545e7c6cdb (diff) | |
download | ydb-832415b6bf33a23a003195441f74d66164038084.tar.gz |
Limit exports & imports count,
15 files changed, 251 insertions, 75 deletions
diff --git a/ydb/core/protos/flat_tx_scheme.proto b/ydb/core/protos/flat_tx_scheme.proto index 4848224b1b..ed2d300245 100644 --- a/ydb/core/protos/flat_tx_scheme.proto +++ b/ydb/core/protos/flat_tx_scheme.proto @@ -179,6 +179,9 @@ message TSchemeLimits { optional uint64 MaxPQPartitions = 14; optional uint64 MaxTableCdcStreams = 15; + + optional uint64 MaxExports = 16; + optional uint64 MaxImports = 17; } message TEvInitTenantSchemeShard { diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 04abd5df59..ae94c7e39d 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -1224,6 +1224,29 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { return true; } + template <typename TRowSet> + TSchemeLimits LoadSchemeLimits(const TSchemeLimits& defaults, TRowSet& rowSet) { + return TSchemeLimits { + .MaxDepth = rowSet.template GetValueOrDefault<Schema::SubDomains::DepthLimit>(defaults.MaxDepth), + .MaxPaths = rowSet.template GetValueOrDefault<Schema::SubDomains::PathsLimit>(defaults.MaxPaths), + .MaxChildrenInDir = rowSet.template GetValueOrDefault<Schema::SubDomains::ChildrenLimit>(defaults.MaxChildrenInDir), + .MaxAclBytesSize = rowSet.template GetValueOrDefault<Schema::SubDomains::AclByteSizeLimit>(defaults.MaxAclBytesSize), + .MaxPathElementLength = rowSet.template GetValueOrDefault<Schema::SubDomains::PathElementLength>(defaults.MaxPathElementLength), + .ExtraPathSymbolsAllowed = rowSet.template GetValueOrDefault<Schema::SubDomains::ExtraPathSymbolsAllowed>(defaults.ExtraPathSymbolsAllowed), + .MaxTableColumns = rowSet.template GetValueOrDefault<Schema::SubDomains::TableColumnsLimit>(defaults.MaxTableColumns), + .MaxTableColumnNameLength = rowSet.template GetValueOrDefault<Schema::SubDomains::TableColumnNameLengthLimit>(defaults.MaxTableColumnNameLength), + .MaxTableKeyColumns = rowSet.template GetValueOrDefault<Schema::SubDomains::TableKeyColumnsLimit>(defaults.MaxTableKeyColumns), + .MaxTableIndices = rowSet.template GetValueOrDefault<Schema::SubDomains::TableIndicesLimit>(defaults.MaxTableIndices), + .MaxTableCdcStreams = rowSet.template GetValueOrDefault<Schema::SubDomains::TableCdcStreamsLimit>(defaults.MaxTableCdcStreams), + .MaxShards = rowSet.template GetValueOrDefault<Schema::SubDomains::ShardsLimit>(defaults.MaxShards), + .MaxShardsInPath = rowSet.template GetValueOrDefault<Schema::SubDomains::PathShardsLimit>(defaults.MaxShardsInPath), + .MaxConsistentCopyTargets = rowSet.template GetValueOrDefault<Schema::SubDomains::ConsistentCopyingTargetsLimit>(defaults.MaxConsistentCopyTargets), + .MaxPQPartitions = rowSet.template GetValueOrDefault<Schema::SubDomains::PQPartitionsLimit>(defaults.MaxPQPartitions), + .MaxExports = rowSet.template GetValueOrDefault<Schema::SubDomains::ExportsLimit>(defaults.MaxExports), + .MaxImports = rowSet.template GetValueOrDefault<Schema::SubDomains::ImportsLimit>(defaults.MaxImports), + }; + } + bool ReadEverything(TTransactionContext& txc, const TActorContext& ctx) { const TOwnerId selfId = Self->TabletID(); @@ -1436,21 +1459,8 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { if (row.IsValid()) { version = row.GetValue<Schema::SubDomains::AlterVersion>(); - rootLimits.MaxDepth = row.GetValueOrDefault<Schema::SubDomains::DepthLimit>(rootLimits.MaxDepth); - rootLimits.MaxPaths = row.GetValueOrDefault<Schema::SubDomains::PathsLimit>(rootLimits.MaxPathsCompat); - rootLimits.MaxChildrenInDir = row.GetValueOrDefault<Schema::SubDomains::ChildrenLimit>(rootLimits.MaxChildrenInDir); - rootLimits.MaxAclBytesSize = row.GetValueOrDefault<Schema::SubDomains::AclByteSizeLimit>(rootLimits.MaxAclBytesSize); - rootLimits.MaxTableColumns = row.GetValueOrDefault<Schema::SubDomains::TableColumnsLimit>(rootLimits.MaxTableColumns); - rootLimits.MaxTableColumnNameLength = row.GetValueOrDefault<Schema::SubDomains::TableColumnNameLengthLimit>(rootLimits.MaxTableColumnNameLength); - rootLimits.MaxTableKeyColumns = row.GetValueOrDefault<Schema::SubDomains::TableKeyColumnsLimit>(rootLimits.MaxTableKeyColumns); - rootLimits.MaxTableIndices = row.GetValueOrDefault<Schema::SubDomains::TableIndicesLimit>(rootLimits.MaxTableIndices); - rootLimits.MaxTableCdcStreams = row.GetValueOrDefault<Schema::SubDomains::TableCdcStreamsLimit>(rootLimits.MaxTableCdcStreams); - rootLimits.MaxShards = row.GetValueOrDefault<Schema::SubDomains::ShardsLimit>(rootLimits.MaxShards); - rootLimits.MaxShardsInPath = row.GetValueOrDefault<Schema::SubDomains::PathShardsLimit>(rootLimits.MaxShardsInPath); - rootLimits.MaxConsistentCopyTargets = row.GetValueOrDefault<Schema::SubDomains::ConsistentCopyingTargetsLimit>(rootLimits.MaxConsistentCopyTargets); - rootLimits.MaxPathElementLength = row.GetValueOrDefault<Schema::SubDomains::PathElementLength>(rootLimits.MaxPathElementLength); - rootLimits.ExtraPathSymbolsAllowed = row.GetValueOrDefault<Schema::SubDomains::ExtraPathSymbolsAllowed>(rootLimits.ExtraPathSymbolsAllowed); - rootLimits.MaxPQPartitions = row.GetValueOrDefault<Schema::SubDomains::PQPartitionsLimit>(rootLimits.MaxPQPartitions); + rootLimits.MaxPaths = rootLimits.MaxPathsCompat; + rootLimits = LoadSchemeLimits(rootLimits, row); } TSubDomainInfo::TPtr rootDomainInfo = new TSubDomainInfo(version, Self->RootPathId()); @@ -1494,24 +1504,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { TTabletId sharedHiveId = rowset.GetValue<Schema::SubDomains::SharedHiveId>(); domainInfo->SetSharedHive(sharedHiveId); - TSchemeLimits limits = rootLimits; - limits.MaxDepth = rowset.GetValueOrDefault<Schema::SubDomains::DepthLimit>(limits.MaxDepth); - limits.MaxPaths = rowset.GetValueOrDefault<Schema::SubDomains::PathsLimit>(limits.MaxPaths); - limits.MaxChildrenInDir = rowset.GetValueOrDefault<Schema::SubDomains::ChildrenLimit>(limits.MaxChildrenInDir); - limits.MaxAclBytesSize = rowset.GetValueOrDefault<Schema::SubDomains::AclByteSizeLimit>(limits.MaxAclBytesSize); - limits.MaxTableColumns = rowset.GetValueOrDefault<Schema::SubDomains::TableColumnsLimit>(limits.MaxTableColumns); - limits.MaxTableColumnNameLength = rowset.GetValueOrDefault<Schema::SubDomains::TableColumnNameLengthLimit>(limits.MaxTableColumnNameLength); - limits.MaxTableKeyColumns = rowset.GetValueOrDefault<Schema::SubDomains::TableKeyColumnsLimit>(limits.MaxTableKeyColumns); - limits.MaxTableIndices = rowset.GetValueOrDefault<Schema::SubDomains::TableIndicesLimit>(limits.MaxTableIndices); - limits.MaxTableCdcStreams = rowset.GetValueOrDefault<Schema::SubDomains::TableCdcStreamsLimit>(limits.MaxTableCdcStreams); - limits.MaxShards = rowset.GetValueOrDefault<Schema::SubDomains::ShardsLimit>(limits.MaxShards); - limits.MaxShardsInPath = rowset.GetValueOrDefault<Schema::SubDomains::PathShardsLimit>(limits.MaxShardsInPath); - limits.MaxConsistentCopyTargets = rowset.GetValueOrDefault<Schema::SubDomains::ConsistentCopyingTargetsLimit>(limits.MaxConsistentCopyTargets); - limits.MaxPathElementLength = rowset.GetValueOrDefault<Schema::SubDomains::PathElementLength>(limits.MaxPathElementLength); - limits.ExtraPathSymbolsAllowed = rowset.GetValueOrDefault<Schema::SubDomains::ExtraPathSymbolsAllowed>(limits.ExtraPathSymbolsAllowed); - limits.MaxPQPartitions = rowset.GetValueOrDefault<Schema::SubDomains::PQPartitionsLimit>(limits.MaxPQPartitions); - - domainInfo->SetSchemeLimits(limits); + domainInfo->SetSchemeLimits(LoadSchemeLimits(rootLimits, rowset)); if (rowset.HaveValue<Schema::SubDomains::DeclaredSchemeQuotas>()) { NKikimrSubDomains::TSchemeQuotas declaredSchemeQuotas; diff --git a/ydb/core/tx/schemeshard/schemeshard__init_root.cpp b/ydb/core/tx/schemeshard/schemeshard__init_root.cpp index 47247d1f95..6d37d4f2e7 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init_root.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init_root.cpp @@ -382,7 +382,7 @@ struct TSchemeShard::TTxInitTenantSchemeShard : public TSchemeShard::TRwTxBase { subdomain->AddStoragePool(x); } - subdomain->SetSchemeLimits(TSchemeLimits(schemeLimits)); + subdomain->SetSchemeLimits(TSchemeLimits::FromProto(schemeLimits)); if (record.HasDeclaredSchemeQuotas()) { subdomain->ApplyDeclaredSchemeQuotas(record.GetDeclaredSchemeQuotas(), ctx.Now()); diff --git a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp index a271c24a27..210c80b84b 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp @@ -72,6 +72,14 @@ struct TSchemeShard::TExport::TTxCreate: public TSchemeShard::TXxport::TTxBase { if (!checks) { return Reply(std::move(response), Ydb::StatusIds::BAD_REQUEST, checks.GetError()); } + + if (!request.HasUserSID() || !Self->SystemBackupSIDs.contains(request.GetUserSID())) { + checks.ExportsLimit(); + } + + if (!checks) { + return Reply(std::move(response), Ydb::StatusIds::PRECONDITION_FAILED, checks.GetError()); + } } TExportInfo::TPtr exportInfo = nullptr; diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index 4ef3a16946..a3a879f1d3 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -74,6 +74,14 @@ struct TSchemeShard::TImport::TTxCreate: public TSchemeShard::TXxport::TTxBase { if (!checks) { return Reply(std::move(response), Ydb::StatusIds::BAD_REQUEST, checks.GetError()); } + + if (!request.HasUserSID() || !Self->SystemBackupSIDs.contains(request.GetUserSID())) { + checks.ImportsLimit(); + } + + if (!checks) { + return Reply(std::move(response), Ydb::StatusIds::PRECONDITION_FAILED, checks.GetError()); + } } TImportInfo::TPtr importInfo = nullptr; diff --git a/ydb/core/tx/schemeshard/schemeshard_path.cpp b/ydb/core/tx/schemeshard/schemeshard_path.cpp index 685e7837a5..b0f8120e9e 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path.cpp @@ -819,6 +819,38 @@ const TPath::TChecker& TPath::TChecker::PathShardsLimit(ui64 delta, EStatus stat << ", delta: " << delta); } +const TPath::TChecker& TPath::TChecker::ExportsLimit(ui64 delta, EStatus status) const { + if (Failed) { + return *this; + } + + TSubDomainInfo::TPtr domainInfo = Path.DomainInfo(); + if (Path.SS->Exports.size() + delta <= domainInfo->GetSchemeLimits().MaxExports) { + return *this; + } + + return Fail(status, TStringBuilder() << "exports count limit exceeded" + << ", limit: " << domainInfo->GetSchemeLimits().MaxExports + << ", exports: " << Path.SS->Exports.size() + << ", delta: " << delta); +} + +const TPath::TChecker& TPath::TChecker::ImportsLimit(ui64 delta, EStatus status) const { + if (Failed) { + return *this; + } + + TSubDomainInfo::TPtr domainInfo = Path.DomainInfo(); + if (Path.SS->Imports.size() + delta <= domainInfo->GetSchemeLimits().MaxImports) { + return *this; + } + + return Fail(status, TStringBuilder() << "imports count limit exceeded" + << ", limit: " << domainInfo->GetSchemeLimits().MaxImports + << ", exports: " << Path.SS->Imports.size() + << ", delta: " << delta); +} + const TPath::TChecker& TPath::TChecker::NotChildren(EStatus status) const { if (Failed) { return *this; diff --git a/ydb/core/tx/schemeshard/schemeshard_path.h b/ydb/core/tx/schemeshard/schemeshard_path.h index 89a76b2867..a258e73011 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path.h +++ b/ydb/core/tx/schemeshard/schemeshard_path.h @@ -86,6 +86,8 @@ public: const TChecker& IsValidACL(const TString& acl, EStatus status = EStatus::StatusInvalidParameter) const; const TChecker& PQPartitionsLimit(ui64 delta = 1, EStatus status = EStatus::StatusResourceExhausted) const; const TChecker& PQReservedStorageLimit(ui64 delta = 1, EStatus status = EStatus::StatusResourceExhausted) const; + const TChecker& ExportsLimit(ui64 delta = 1, EStatus status = EStatus::StatusResourceExhausted) const; + const TChecker& ImportsLimit(ui64 delta = 1, EStatus status = EStatus::StatusResourceExhausted) const; const TChecker& IsExternalTable(EStatus status = EStatus::StatusNameConflict) const; const TChecker& IsExternalDataSource(EStatus status = EStatus::StatusNameConflict) const; }; diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h index 723ecda8b7..9f69cbe17c 100644 --- a/ydb/core/tx/schemeshard/schemeshard_schema.h +++ b/ydb/core/tx/schemeshard/schemeshard_schema.h @@ -725,6 +725,8 @@ struct Schema : NIceDb::Schema { struct DiskQuotaExceeded : Column<25, NScheme::NTypeIds::Bool> {}; struct SecurityStateVersion : Column<26, NScheme::NTypeIds::Uint64> {}; struct TableCdcStreamsLimit : Column<27, NScheme::NTypeIds::Uint64> {}; + struct ExportsLimit : Column<28, NScheme::NTypeIds::Uint64> {}; + struct ImportsLimit : Column<29, NScheme::NTypeIds::Uint64> {}; using TKey = TableKey<PathId>; using TColumns = TableColumns< @@ -754,7 +756,9 @@ struct Schema : NIceDb::Schema { StateVersion, DiskQuotaExceeded, SecurityStateVersion, - TableCdcStreamsLimit + TableCdcStreamsLimit, + ExportsLimit, + ImportsLimit >; }; diff --git a/ydb/core/tx/schemeshard/schemeshard_types.cpp b/ydb/core/tx/schemeshard/schemeshard_types.cpp index cc3c439f5f..443cafd3e7 100644 --- a/ydb/core/tx/schemeshard/schemeshard_types.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_types.cpp @@ -2,52 +2,62 @@ namespace NKikimr::NSchemeShard { -TSchemeLimits::TSchemeLimits(const NKikimrScheme::TSchemeLimits& proto) { +TSchemeLimits TSchemeLimits::FromProto(const NKikimrScheme::TSchemeLimits& proto) { + TSchemeLimits result; + if (proto.HasMaxDepth()) { - MaxDepth = proto.GetMaxDepth(); + result.MaxDepth = proto.GetMaxDepth(); } if (proto.HasMaxPaths()) { - MaxPaths = proto.GetMaxPaths(); + result.MaxPaths = proto.GetMaxPaths(); } if (proto.HasMaxChildrenInDir()) { - MaxChildrenInDir = proto.GetMaxChildrenInDir(); + result.MaxChildrenInDir = proto.GetMaxChildrenInDir(); } if (proto.HasMaxAclBytesSize()) { - MaxAclBytesSize = proto.GetMaxAclBytesSize(); + result.MaxAclBytesSize = proto.GetMaxAclBytesSize(); } if (proto.HasMaxTableColumns()) { - MaxTableColumns = proto.GetMaxTableColumns(); + result.MaxTableColumns = proto.GetMaxTableColumns(); } if (proto.HasMaxTableColumnNameLength()) { - MaxTableColumnNameLength = proto.GetMaxTableColumnNameLength(); + result.MaxTableColumnNameLength = proto.GetMaxTableColumnNameLength(); } if (proto.HasMaxTableKeyColumns()) { - MaxTableKeyColumns = proto.GetMaxTableKeyColumns(); + result.MaxTableKeyColumns = proto.GetMaxTableKeyColumns(); } if (proto.HasMaxTableIndices()) { - MaxTableIndices = proto.GetMaxTableIndices(); + result.MaxTableIndices = proto.GetMaxTableIndices(); } if (proto.HasMaxTableCdcStreams()) { - MaxTableCdcStreams = proto.GetMaxTableCdcStreams(); + result.MaxTableCdcStreams = proto.GetMaxTableCdcStreams(); } if (proto.HasMaxShards()) { - MaxShards = proto.GetMaxShards(); + result.MaxShards = proto.GetMaxShards(); } if (proto.HasMaxShardsInPath()) { - MaxShardsInPath = proto.GetMaxShardsInPath(); + result.MaxShardsInPath = proto.GetMaxShardsInPath(); } if (proto.HasMaxConsistentCopyTargets()) { - MaxConsistentCopyTargets = proto.GetMaxConsistentCopyTargets(); + result.MaxConsistentCopyTargets = proto.GetMaxConsistentCopyTargets(); } if (proto.HasMaxPathElementLength()) { - MaxPathElementLength = proto.GetMaxPathElementLength(); + result.MaxPathElementLength = proto.GetMaxPathElementLength(); } if (proto.HasExtraPathSymbolsAllowed()) { - ExtraPathSymbolsAllowed = proto.GetExtraPathSymbolsAllowed(); + result.ExtraPathSymbolsAllowed = proto.GetExtraPathSymbolsAllowed(); } if (proto.HasMaxPQPartitions()) { - MaxPQPartitions = proto.GetMaxPQPartitions(); + result.MaxPQPartitions = proto.GetMaxPQPartitions(); + } + if (proto.HasMaxExports()) { + result.MaxExports = proto.GetMaxExports(); + } + if (proto.HasMaxImports()) { + result.MaxImports = proto.GetMaxImports(); } + + return result; } NKikimrScheme::TSchemeLimits TSchemeLimits::AsProto() const { @@ -72,6 +82,9 @@ NKikimrScheme::TSchemeLimits TSchemeLimits::AsProto() const { result.SetMaxPQPartitions(MaxPQPartitions); + result.SetMaxExports(MaxExports); + result.SetMaxImports(MaxImports); + return result; } diff --git a/ydb/core/tx/schemeshard/schemeshard_types.h b/ydb/core/tx/schemeshard/schemeshard_types.h index e2dfbe42c9..151b01e4e8 100644 --- a/ydb/core/tx/schemeshard/schemeshard_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_types.h @@ -48,9 +48,11 @@ struct TSchemeLimits { // pq group ui64 MaxPQPartitions = 1000000; - TSchemeLimits() = default; - explicit TSchemeLimits(const NKikimrScheme::TSchemeLimits& proto); + // export & import + ui64 MaxExports = 10; + ui64 MaxImports = 10; + static TSchemeLimits FromProto(const NKikimrScheme::TSchemeLimits& proto); NKikimrScheme::TSchemeLimits AsProto() const; }; diff --git a/ydb/core/tx/schemeshard/ut_export.cpp b/ydb/core/tx/schemeshard/ut_export.cpp index c8d97aafc0..907e564aef 100644 --- a/ydb/core/tx/schemeshard/ut_export.cpp +++ b/ydb/core/tx/schemeshard/ut_export.cpp @@ -90,16 +90,26 @@ namespace { } for (const auto& table : tables) { - TestCreateTable(runtime, schemeshardId, ++txId, dbName, table); + TestCreateTable(runtime, schemeshardId, ++txId, dbName, table, { + NKikimrScheme::StatusAccepted, + NKikimrScheme::StatusAlreadyExists, + }); env.TestWaitNotification(runtime, txId, schemeshardId); } runtime.SetLogPriority(NKikimrServices::DATASHARD_BACKUP, NActors::NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::EXPORT, NActors::NLog::PRI_TRACE); - TestExport(runtime, schemeshardId, ++txId, dbName, request, userSID); + const auto initialStatus = expectedStatus == Ydb::StatusIds::PRECONDITION_FAILED + ? expectedStatus + : Ydb::StatusIds::SUCCESS; + TestExport(runtime, schemeshardId, ++txId, dbName, request, userSID, initialStatus); env.TestWaitNotification(runtime, txId, schemeshardId); + if (initialStatus != Ydb::StatusIds::SUCCESS) { + return; + } + const ui64 exportId = txId; TestGetExport(runtime, schemeshardId, exportId, dbName, expectedStatus); @@ -1060,4 +1070,42 @@ partitioning_settings { env.TestWaitNotification(runtime, exportId); TestGetExport(runtime, exportId, "/MyRoot", Ydb::StatusIds::SUCCESS); } + + Y_UNIT_TEST(ShouldCheckQuotas) { + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TS3Mock s3Mock({}, TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + + const TString userSID = "user@builtin"; + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().SystemBackupSIDs({userSID})); + + TSchemeLimits lowLimits; + lowLimits.MaxExports = 0; + SetSchemeshardSchemaLimits(runtime, lowLimits); + + const TVector<TString> tables = { + R"( + Name: "Table" + Columns { Name: "key" Type: "Utf8" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )", + }; + const TString request = Sprintf(R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_path: "/MyRoot/Table" + destination_prefix: "" + } + } + )", port); + + Run(runtime, env, tables, request, Ydb::StatusIds::PRECONDITION_FAILED); + Run(runtime, env, tables, request, Ydb::StatusIds::SUCCESS, "/MyRoot", false, userSID); + } } diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp index bb4420b8f7..9f66ddeff9 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp @@ -1097,29 +1097,34 @@ namespace NSchemeShardUT_Private { return TestForgetExport(runtime, TTestTxConfig::SchemeShard, txId, dbName, exportId, expectedStatus); } - void AsyncImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr) { + void AsyncImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID) { NKikimrImport::TCreateImportRequest request; UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(requestStr, &request)); - AsyncSend(runtime, schemeshardId, new TEvImport::TEvCreateImportRequest(id, dbName, request)); + auto ev = MakeHolder<TEvImport::TEvCreateImportRequest>(id, dbName, request); + if (userSID) { + ev->Record.SetUserSID(userSID); + } + + AsyncSend(runtime, schemeshardId, ev.Release()); } - void AsyncImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr) { - AsyncImport(runtime, TTestTxConfig::SchemeShard, id, dbName, requestStr); + void AsyncImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID) { + AsyncImport(runtime, TTestTxConfig::SchemeShard, id, dbName, requestStr, userSID); } - void TestImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, + void TestImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID, Ydb::StatusIds::StatusCode expectedStatus) { - AsyncImport(runtime, schemeshardId, id, dbName, requestStr); + AsyncImport(runtime, schemeshardId, id, dbName, requestStr, userSID); TAutoPtr<IEventHandle> handle; auto ev = runtime.GrabEdgeEvent<TEvImport::TEvCreateImportResponse>(handle); UNIT_ASSERT_EQUAL(ev->Record.GetResponse().GetEntry().GetStatus(), expectedStatus); } - void TestImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, + void TestImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID, Ydb::StatusIds::StatusCode expectedStatus) { - TestImport(runtime, TTestTxConfig::SchemeShard, id, dbName, requestStr, expectedStatus); + TestImport(runtime, TTestTxConfig::SchemeShard, id, dbName, requestStr, userSID, expectedStatus); } NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, @@ -1491,14 +1496,17 @@ namespace NSchemeShardUT_Private { (let maxPathLength '('PathElementLength (Uint64 '%lu))) (let extraSymbols '('ExtraPathSymbolsAllowed (Utf8 '"%s"))) (let pqPartitions '('PQPartitionsLimit (Uint64 '%lu))) - (let ret (AsList (UpdateRow 'SubDomains key '(depth paths child acl columns colName keyCols indices streams shards pathShards consCopy maxPathLength extraSymbols pqPartitions)))) + (let exports '('ExportsLimit (Uint64 '%lu))) + (let imports '('ImportsLimit (Uint64 '%lu))) + (let ret (AsList (UpdateRow 'SubDomains key '(depth paths child acl columns colName keyCols indices streams shards pathShards consCopy maxPathLength extraSymbols pqPartitions exports imports)))) (return ret) ) )", domainId, limits.MaxDepth, limits.MaxPaths, limits.MaxChildrenInDir, limits.MaxAclBytesSize, limits.MaxTableColumns, limits.MaxTableColumnNameLength, limits.MaxTableKeyColumns, limits.MaxTableIndices, limits.MaxTableCdcStreams, limits.MaxShards, limits.MaxShardsInPath, limits.MaxConsistentCopyTargets, - limits.MaxPathElementLength, escapedStr.c_str(), limits.MaxPQPartitions); + limits.MaxPathElementLength, escapedStr.c_str(), limits.MaxPQPartitions, + limits.MaxExports, limits.MaxImports); Cdbg << prog << "\n"; NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, schemeShard, prog, result, err); Cdbg << result << "\n"; diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.h b/ydb/core/tx/schemeshard/ut_helpers/helpers.h index f8ff2c09cd..ed822ab77b 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/helpers.h +++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.h @@ -373,11 +373,11 @@ namespace NSchemeShardUT_Private { Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); ////////// import - void AsyncImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr); - void AsyncImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr); - void TestImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, + void AsyncImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID = ""); + void AsyncImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID = ""); + void TestImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID = "", Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); - void TestImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, + void TestImport(TTestActorRuntime& runtime, ui64 id, const TString& dbName, const TString& requestStr, const TString& userSID = "", Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS); NKikimrImport::TEvGetImportResponse TestGetImport(TTestActorRuntime& runtime, ui64 schemeshardId, ui64 id, const TString& dbName, const TVector<Ydb::StatusIds::StatusCode>& expectedStatuses); diff --git a/ydb/core/tx/schemeshard/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore.cpp index 210062406f..4963b7eeed 100644 --- a/ydb/core/tx/schemeshard/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore.cpp @@ -1477,8 +1477,8 @@ Y_UNIT_TEST_SUITE(TImportTests) { void Run(TTestBasicRuntime& runtime, TTestEnv& env, THashMap<TString, TString>&& data, const TString& request, Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS, - const TString& dbName = "/MyRoot", bool serverless = false) { - + const TString& dbName = "/MyRoot", bool serverless = false, const TString& userSID = "") + { ui64 id = 100; TPortManager portManager; @@ -1557,18 +1557,25 @@ Y_UNIT_TEST_SUITE(TImportTests) { runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::IMPORT, NActors::NLog::PRI_TRACE); - TestImport(runtime, schemeshardId, ++id, dbName, Sprintf(request.data(), port)); + const auto initialStatus = expectedStatus == Ydb::StatusIds::PRECONDITION_FAILED + ? expectedStatus + : Ydb::StatusIds::SUCCESS; + TestImport(runtime, schemeshardId, ++id, dbName, Sprintf(request.data(), port), userSID, initialStatus); env.TestWaitNotification(runtime, id, schemeshardId); + if (initialStatus != Ydb::StatusIds::SUCCESS) { + return; + } + TestGetImport(runtime, schemeshardId, id, dbName, expectedStatus); } void Run(TTestBasicRuntime& runtime, THashMap<TString, TString>&& data, const TString& request, Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS, - const TString& dbName = "/MyRoot", bool serverless = false) { + const TString& dbName = "/MyRoot", bool serverless = false, const TString& userSID = "") { TTestEnv env(runtime, TTestEnvOptions()); - Run(runtime, env, std::move(data), request, expectedStatus, dbName, serverless); + Run(runtime, env, std::move(data), request, expectedStatus, dbName, serverless, userSID); } Y_UNIT_TEST(ShouldSucceedOnSingleShardTable) { @@ -2322,6 +2329,42 @@ Y_UNIT_TEST_SUITE(TImportTests) { .GetTransaction(0).GetOperationType() == NKikimrSchemeOp::ESchemeOpApplyIndexBuild; }); } + + Y_UNIT_TEST(ShouldCheckQuotas) { + const TString userSID = "user@builtin"; + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().SystemBackupSIDs({userSID})); + + TSchemeLimits lowLimits; + lowLimits.MaxImports = 0; + SetSchemeshardSchemaLimits(runtime, lowLimits); + + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )", {{"a", 1}}); + + const TString request = R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )"; + + Run(runtime, env, ConvertTestData(data), request, Ydb::StatusIds::PRECONDITION_FAILED); + Run(runtime, env, ConvertTestData(data), request, Ydb::StatusIds::SUCCESS, "/MyRoot", false, userSID); + } } Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema index c20f1d8ce5..487ed9dad6 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema @@ -1241,6 +1241,11 @@ ], "ColumnsAdded": [ { + "ColumnId": 29, + "ColumnName": "ImportsLimit", + "ColumnType": "Uint64" + }, + { "ColumnId": 1, "ColumnName": "PathId", "ColumnType": "Uint64" @@ -1374,12 +1379,18 @@ "ColumnId": 27, "ColumnName": "TableCdcStreamsLimit", "ColumnType": "Uint64" + }, + { + "ColumnId": 28, + "ColumnName": "ExportsLimit", + "ColumnType": "Uint64" } ], "ColumnsDropped": [], "ColumnFamilies": { "0": { "Columns": [ + 29, 1, 2, 3, @@ -1406,7 +1417,8 @@ 24, 25, 26, - 27 + 27, + 28 ], "RoomID": 0, "Codec": 0, |