Backward compatability & fixes

6 files changed, 93 insertions, 46 deletions

diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp
index 293bcdfffa4..f16c47eaf48 100644
--- a/ydb/core/tx/schemeshard/schemeshard__init.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp
@@ -1535,7 +1535,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
                 if (row.IsValid()) {
                     version = row.GetValue<Schema::SubDomains::AlterVersion>();
                     rootLimits.MaxDepth = row.GetValueOrDefault<Schema::SubDomains::DepthLimit>(rootLimits.MaxDepth);
-                    rootLimits.MaxPaths = row.GetValueOrDefault<Schema::SubDomains::PathsLimit>(rootLimits.MaxPaths);
+                    rootLimits.MaxPaths = row.GetValueOrDefault<Schema::SubDomains::PathsLimit>(rootLimits.MaxPathsCompat);
                     rootLimits.MaxChildrenInDir = row.GetValueOrDefault<Schema::SubDomains::ChildrenLimit>(rootLimits.MaxChildrenInDir);
                     rootLimits.MaxAclBytesSize = row.GetValueOrDefault<Schema::SubDomains::AclByteSizeLimit>(rootLimits.MaxAclBytesSize);
                     rootLimits.MaxTableColumns = row.GetValueOrDefault<Schema::SubDomains::TableColumnsLimit>(rootLimits.MaxTableColumns);
diff --git a/ydb/core/tx/schemeshard/schemeshard__init_root.cpp b/ydb/core/tx/schemeshard/schemeshard__init_root.cpp
index e8d12429402..117c6bd91f9 100644
--- a/ydb/core/tx/schemeshard/schemeshard__init_root.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__init_root.cpp
@@ -132,6 +132,7 @@ struct TSchemeShard::TTxInitRoot : public TSchemeShard::TRwTxBase {
         Self->PersistUpdateNextPathId(db);
         Self->PersistUpdateNextShardIdx(db);
         Self->PersistStoragePools(db, Self->RootPathId(), *newDomain);
+        Self->PersistSchemeLimit(db, Self->RootPathId(), *newDomain);
         Self->PersistACL(db, newPath);
 
         Self->InitState = TTenantInitState::Done;
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp
index 99ea34bf603..1410a10f61f 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp
@@ -1,13 +1,14 @@
 #include "schemeshard__operation_part.h"
 #include "schemeshard__operation_common.h"
-#include "schemeshard_path_element.h"
-
 #include "schemeshard_impl.h"
+#include "schemeshard_path_element.h"
 
 #include <ydb/core/base/path.h>
 #include <ydb/core/protos/flat_tx_scheme.pb.h>
 #include <ydb/core/protos/flat_scheme_op.pb.h>
 
+#include <util/generic/algorithm.h>
+
 NKikimrSchemeOp::TModifyScheme CopyTableTask(NKikimr::NSchemeShard::TPath& src, NKikimr::NSchemeShard::TPath& dst, bool omitFollowers, bool isBackup) {
     using namespace NKikimr::NSchemeShard;
 
@@ -23,7 +24,6 @@ NKikimrSchemeOp::TModifyScheme CopyTableTask(NKikimr::NSchemeShard::TPath& src,
     return scheme;
 }
 
-
 NKikimrSchemeOp::TModifyScheme CreateIndexTask(NKikimr::NSchemeShard::TTableIndexInfo::TPtr indexInfo, NKikimr::NSchemeShard::TPath& dst) {
     using namespace NKikimr::NSchemeShard;
 
@@ -45,21 +45,19 @@ NKikimrSchemeOp::TModifyScheme CreateIndexTask(NKikimr::NSchemeShard::TTableInde
     return scheme;
 }
 
-
-namespace NKikimr {
-namespace NSchemeShard {
+namespace NKikimr::NSchemeShard {
 
 TVector<ISubOperationBase::TPtr> CreateConsistentCopyTables(TOperationId nextId, const TTxTransaction& tx, TOperationContext& context) {
     Y_VERIFY(tx.GetOperationType() == NKikimrSchemeOp::EOperationType::ESchemeOpCreateConsistentCopyTables);
 
-    auto consistentCopying = tx.GetCreateConsistentCopyTables();
+    const auto& op = tx.GetCreateConsistentCopyTables();
 
-    if (0 == consistentCopying.CopyTableDescriptionsSize()) {
+    if (0 == op.CopyTableDescriptionsSize()) {
         TString msg = TStringBuilder() << "no task to do, empty list CopyTableDescriptions";
         return {CreateReject(nextId, NKikimrScheme::EStatus::StatusInvalidParameter, msg)};
     }
 
-    TPath firstPath = TPath::Resolve(consistentCopying.GetCopyTableDescriptions(0).GetSrcPath(), context.SS);
+    TPath firstPath = TPath::Resolve(op.GetCopyTableDescriptions(0).GetSrcPath(), context.SS);
     {
         auto checks = TPath::TChecker(firstPath);
         checks
@@ -72,26 +70,33 @@ TVector<ISubOperationBase::TPtr> CreateConsistentCopyTables(TOperationId nextId,
         }
     }
 
-    TSubDomainInfo::TPtr domainInfo = firstPath.DomainInfo();
-    if (consistentCopying.CopyTableDescriptionsSize() > domainInfo->GetSchemeLimits().MaxConsistentCopyTargets) {
-        auto msg = TStringBuilder() << "Targets count has reached maximum value"
-                                    << ", limit for consistent copying in domain: " << domainInfo->GetSchemeLimits().MaxConsistentCopyTargets
-                                    << ", intention to consistent copy: " << consistentCopying.CopyTableDescriptionsSize();
-        return {CreateReject(nextId, NKikimrScheme::EStatus::StatusInvalidParameter, msg)};
+    const auto allForBackup = AllOf(op.GetCopyTableDescriptions(), [](const auto& item) {
+        return item.GetIsBackup();
+    });
+
+    const auto& limits = firstPath.DomainInfo()->GetSchemeLimits();
+    const auto limit = allForBackup
+        ? Max(limits.MaxObjectsInBackup, limits.MaxConsistentCopyTargets)
+        : limits.MaxConsistentCopyTargets;
+
+    if (op.CopyTableDescriptionsSize() > limit) {
+        return {CreateReject(nextId, NKikimrScheme::EStatus::StatusInvalidParameter, TStringBuilder()
+            << "Consistent copy object count limit exceeded"
+                << ", limit: " << limit
+                << ", objects: " << op.CopyTableDescriptionsSize()
+        )};
     }
 
-    {
-        TString errStr;
-        if (!context.SS->CheckApplyIf(tx, errStr)) {
-            return {CreateReject(nextId, NKikimrScheme::EStatus::StatusPreconditionFailed, errStr)};
-        }
+    TString errStr;
+    if (!context.SS->CheckApplyIf(tx, errStr)) {
+        return {CreateReject(nextId, NKikimrScheme::EStatus::StatusPreconditionFailed, errStr)};
     }
 
     TVector<ISubOperationBase::TPtr> result;
 
-    for (auto& descr: consistentCopying.GetCopyTableDescriptions()) {
-        auto& srcStr = descr.GetSrcPath();
-        auto& dstStr = descr.GetDstPath();
+    for (const auto& descr: op.GetCopyTableDescriptions()) {
+        const auto& srcStr = descr.GetSrcPath();
+        const auto& dstStr = descr.GetDstPath();
 
         TPath srcPath = TPath::Resolve(srcStr, context.SS);
         {
@@ -110,9 +115,8 @@ TVector<ISubOperationBase::TPtr> CreateConsistentCopyTables(TOperationId nextId,
         TPath dstPath = TPath::Resolve(dstStr, context.SS);
         TPath dstParentPath = dstPath.Parent();
 
-        result.push_back(CreateCopyTable(TOperationId(nextId.GetTxId(),
-                                                      nextId.GetSubTxId() + result.size()),
-                                         CopyTableTask(srcPath, dstPath, descr.GetOmitFollowers(), descr.GetIsBackup())));
+        result.push_back(CreateCopyTable(NextPartId(nextId, result),
+            CopyTableTask(srcPath, dstPath, descr.GetOmitFollowers(), descr.GetIsBackup())));
 
         if (descr.GetOmitIndexes()) {
             continue;
@@ -153,4 +157,3 @@ TVector<ISubOperationBase::TPtr> CreateConsistentCopyTables(TOperationId nextId,
 }
 
 }
-}
diff --git a/ydb/core/tx/schemeshard/schemeshard_types.h b/ydb/core/tx/schemeshard/schemeshard_types.h
index a7d9a76fdf9..dfb3bdcedfb 100644
--- a/ydb/core/tx/schemeshard/schemeshard_types.h
+++ b/ydb/core/tx/schemeshard/schemeshard_types.h
@@ -10,9 +10,13 @@
 namespace NKikimr::NSchemeShard {
 
 struct TSchemeLimits {
+    // Used for backward compatability in case of old databases without explicit limits
+    static constexpr ui64 MaxPathsCompat = 200*1000;
+    static constexpr ui64 MaxObjectsInBackup = 10*1000;
+
     // path
     ui64 MaxDepth = 32;
-    ui64 MaxPaths = 10*1000;
+    ui64 MaxPaths = MaxObjectsInBackup;
     ui64 MaxChildrenInDir = 100*1000;
     ui64 MaxAclBytesSize = 10 << 10;
     ui64 MaxPathElementLength = 255;
@@ -26,7 +30,7 @@ struct TSchemeLimits {
     ui64 MaxTableCdcStreams = 5;
     ui64 MaxShards = 200*1000; // In each database
     ui64 MaxShardsInPath = 35*1000; // In each path in database
-    ui64 MaxConsistentCopyTargets = 10*1000; // by default same as MaxPaths
+    ui64 MaxConsistentCopyTargets = MaxObjectsInBackup;
 
     // pq group
     ui64 MaxPQPartitions = 1000000;
diff --git a/ydb/core/tx/schemeshard/ut_base.cpp b/ydb/core/tx/schemeshard/ut_base.cpp
index 7d3f2bb600d..9989f500c9c 100644
--- a/ydb/core/tx/schemeshard/ut_base.cpp
+++ b/ydb/core/tx/schemeshard/ut_base.cpp
@@ -613,18 +613,15 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) {
 
         env.TestWaitNotification(runtime, {txId, txId-1, txId-2, txId-3});
 
-        TestUserAttrs(runtime, ++txId, "/", "MyRoot", AlterUserAttrs({{"__extra_path_symbols_allowed", "_.-"}}));
-        env.TestWaitNotification(runtime, txId);
-        RebootTablet(runtime, TTestTxConfig::SchemeShard, runtime.AllocateEdgeActor());
+        TSchemeLimits lowLimits;
+        lowLimits.ExtraPathSymbolsAllowed = "_.-";
+        SetSchemeshardSchemaLimits(runtime, lowLimits);
 
         TestMkDir(runtime, ++txId, "/MyRoot", "Dir1!", {NKikimrScheme::StatusSchemeError});
         TestMkDir(runtime, ++txId, "/MyRoot", "Dir1?", {NKikimrScheme::StatusSchemeError});
         TestMkDir(runtime, ++txId, "/MyRoot", "Dir1@", {NKikimrScheme::StatusSchemeError});
         TestMkDir(runtime, ++txId, "/MyRoot", "Dir1:", {NKikimrScheme::StatusSchemeError});
 
-
-        TSchemeLimits lowLimits;
-
         lowLimits.ExtraPathSymbolsAllowed = "!?@:";
         SetSchemeshardSchemaLimits(runtime, lowLimits);
 
@@ -639,9 +636,6 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) {
         lowLimits.ExtraPathSymbolsAllowed = "!";
         SetSchemeshardSchemaLimits(runtime, lowLimits);
 
-        TestDescribeResult(DescribePath(runtime, "/MyRoot"),
-                           {NLs::UserAttrsEqual({{"__extra_path_symbols_allowed", "_.-"}})});
-
         TestDescribeResult(DescribePath(runtime, "/MyRoot/Dir1!"),
                            {NLs::Finished,
                             NLs::PathVersionEqual(3)});
@@ -1419,9 +1413,9 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) {
          )", {NKikimrScheme::StatusPathDoesNotExist});
 
         {
-            TestUserAttrs(runtime, ++txId, "/", "MyRoot", AlterUserAttrs({{"__extra_path_symbols_allowed", "-_."}}));
-            env.TestWaitNotification(runtime, txId);
-            RebootTablet(runtime, TTestTxConfig::SchemeShard, runtime.AllocateEdgeActor());
+            TSchemeLimits lowLimits;
+            lowLimits.ExtraPathSymbolsAllowed = "-_.";
+            SetSchemeshardSchemaLimits(runtime, lowLimits);
 
             TestConsistentCopyTables(runtime, ++txId, "/", R"(
                            CopyTableDescriptions {
@@ -3725,6 +3719,54 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) {
         env.TestWaitNotification(runtime, txId);
     }
 
+    Y_UNIT_TEST(ConsistentCopyTablesForBackup) {
+        TTestBasicRuntime runtime;
+        TTestEnv env(runtime);
+        ui64 txId = 100;
+
+        TSchemeLimits limits;
+        limits.MaxConsistentCopyTargets = 1; // should not affect
+        SetSchemeshardSchemaLimits(runtime, limits);
+
+        // create two tables
+        for (int i = 1; i <= 2; ++i) {
+            TestCreateTable(runtime, ++txId, "/MyRoot", Sprintf(R"(
+                Name: "Table%i"
+                Columns { Name: "key" Type: "Uint32"}
+                Columns { Name: "value" Type: "Utf8"}
+                KeyColumnNames: ["key"]
+            )", i));
+            env.TestWaitNotification(runtime, txId);
+        }
+
+        // negative
+        TestConsistentCopyTables(runtime, ++txId, "/", R"(
+            CopyTableDescriptions {
+                SrcPath: "/MyRoot/Table1"
+                DstPath: "/MyRoot/CopyTable1"
+            }
+            CopyTableDescriptions {
+                SrcPath: "/MyRoot/Table2"
+                DstPath: "/MyRoot/CopyTable2"
+            }
+        )", {NKikimrScheme::StatusInvalidParameter});
+
+        // positive
+        TestConsistentCopyTables(runtime, ++txId, "/", R"(
+            CopyTableDescriptions {
+                SrcPath: "/MyRoot/Table1"
+                DstPath: "/MyRoot/CopyTable1"
+                IsBackup: true
+            }
+            CopyTableDescriptions {
+                SrcPath: "/MyRoot/Table2"
+                DstPath: "/MyRoot/CopyTable2"
+                IsBackup: true
+            }
+        )");
+        env.TestWaitNotification(runtime, txId);
+    }
+
     Y_UNIT_TEST(CreateIndexedTableAfterBackup) {
         TTestBasicRuntime runtime;
         TTestEnv env(runtime);
diff --git a/ydb/core/tx/schemeshard/ut_extsubdomain.cpp b/ydb/core/tx/schemeshard/ut_extsubdomain.cpp
index 78d148ab10e..159885ef3aa 100644
--- a/ydb/core/tx/schemeshard/ut_extsubdomain.cpp
+++ b/ydb/core/tx/schemeshard/ut_extsubdomain.cpp
@@ -351,10 +351,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardExtSubDomainTest) {
         TSchemeLimits lowLimits;
         lowLimits.MaxPathElementLength = 10;
         lowLimits.ExtraPathSymbolsAllowed = ".-";
-
-        TestUserAttrs(runtime, ++txId, "/", "MyRoot", AlterUserAttrs({{"__extra_path_symbols_allowed", lowLimits.ExtraPathSymbolsAllowed}}));
-        env.TestWaitNotification(runtime, txId);
-        RebootTablet(runtime, TTestTxConfig::SchemeShard, runtime.AllocateEdgeActor());
+        SetSchemeshardSchemaLimits(runtime, lowLimits);
 
         TestCreateExtSubDomain(runtime, ++txId,  "/MyRoot",
             R"(Name: "USER+0")",