change default shards count in column tables: 1 -> 64

3 files changed, 132 insertions, 23 deletions

diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_table.cpp
index 9c5fd0995c..c6c0ddd84b 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_table.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_table.cpp
@@ -14,10 +14,14 @@ namespace NKikimr::NSchemeShard {
 namespace {
 
 class TTableConstructorBase {
-    YDB_READONLY(ui32, ShardsCount, 1);
-    YDB_READONLY_DEF(TString, Name);
-    YDB_OPT(NKikimrSchemeOp::TColumnDataLifeCycle, TtlSettings);
-    YDB_OPT(NKikimrSchemeOp::TColumnTableSharding, Sharding);
+public:
+    static constexpr ui32 DEFAULT_SHARDS_COUNT = 64;
+
+private:
+    ui32 ShardsCount = 0;
+    TString Name;
+    std::optional<NKikimrSchemeOp::TColumnDataLifeCycle> TtlSettings;
+    std::optional<NKikimrSchemeOp::TColumnTableSharding> Sharding;
 
 public:
     bool Deserialize(const NKikimrSchemeOp::TColumnTableDescription& description, IErrorCollector& errors) {
@@ -32,14 +36,12 @@ public:
             return false;
         }
 
-        ShardsCount = Max(ui32(1), description.GetColumnShardCount());
+        ShardsCount = description.GetColumnShardCount();
+
         if (description.HasSharding()) {
             Sharding = description.GetSharding();
-        }
-
-        if (!Sharding && ShardsCount > 1) {
-            errors.AddError("Sharding is not set");
-            return false;
+        } else {
+            Sharding = DefaultSharding();
         }
 
         if (!DoDeserialize(description, errors)) {
@@ -62,8 +64,8 @@ public:
         BuildDescription(tableInfo->Description);
         tableInfo->Description.SetColumnShardCount(ShardsCount);
         tableInfo->Description.SetName(Name);
-        if (HasTtlSettings()) {
-            tableInfo->Description.MutableTtlSettings()->CopyFrom(GetTtlSettingsUnsafe());
+        if (TtlSettings) {
+            tableInfo->Description.MutableTtlSettings()->CopyFrom(*TtlSettings);
             tableInfo->Description.MutableTtlSettings()->SetVersion(1);
         }
         if (!SetSharding(GetSchema(), tableInfo, errors)) {
@@ -78,12 +80,15 @@ private:
     virtual const TOlapSchema& GetSchema() const = 0;
 
     bool SetSharding(const TOlapSchema& schema, TColumnTableInfo::TPtr tableInfo, IErrorCollector& errors) const {
-        if (Sharding) {
-            tableInfo->Sharding = *Sharding;
-        } else {
-            Y_VERIFY(ShardsCount == 1);
-            tableInfo->Sharding.MutableRandomSharding();
-        };
+        if (!ShardsCount) {
+            errors.AddError("Shards count is zero");
+            return false;
+        }
+        if (!Sharding) {
+            errors.AddError("Sharding is not set");
+            return false;
+        }
+        tableInfo->Sharding = *Sharding;
 
         switch (tableInfo->Sharding.Method_case()) {
             case NKikimrSchemeOp::TColumnTableSharding::kRandomSharding: {
@@ -96,7 +101,10 @@ private:
             case NKikimrSchemeOp::TColumnTableSharding::kHashSharding: {
                 auto& sharding = *tableInfo->Sharding.MutableHashSharding();
                 if (sharding.ColumnsSize() == 0) {
-                    errors.AddError(Sprintf("Hash sharding requires a non-empty list of columns"));
+                    sharding.MutableColumns()->CopyFrom(tableInfo->Description.GetSchema().GetKeyColumnNames());
+                }
+                if (ShardsCount > 1 && sharding.ColumnsSize() == 0) {
+                    errors.AddError("Hash sharding requires a non-empty list of columns or primary key specified");
                     return false;
                 }
                 for (const TString& columnName : sharding.GetColumns()) {
@@ -121,13 +129,20 @@ private:
         }
         return true;
     }
+
+    static NKikimrSchemeOp::TColumnTableSharding DefaultSharding() {
+        NKikimrSchemeOp::TColumnTableSharding sharding;
+        auto* hashSharding = sharding.MutableHashSharding();
+        hashSharding->SetFunction(NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_MODULO_N);
+        return sharding;
+    }
 };
 
 class TOlapPresetConstructor : public TTableConstructorBase {
-    YDB_READONLY(ui32, PresetId, 0);
-    YDB_READONLY(TString, PresetName, "default");
-
+    ui32 PresetId = 0;
+    TString PresetName = "default";
     const TOlapStoreInfo& StoreInfo;
+
 public:
     TOlapPresetConstructor(const TOlapStoreInfo& storeInfo)
         : StoreInfo(storeInfo)
@@ -552,7 +567,12 @@ public:
 
         const auto acceptExisted = !Transaction.GetFailOnExist();
         const TString& parentPathStr = Transaction.GetWorkingDir();
-        auto& createDescription = Transaction.GetCreateColumnTable();
+
+        // Copy CreateColumnTable for changes. Update defaut sharding if not set.
+        auto createDescription = Transaction.GetCreateColumnTable();
+        if (!createDescription.HasColumnShardCount()) {
+            createDescription.SetColumnShardCount(TTableConstructorBase::DEFAULT_SHARDS_COUNT);
+        }
         const TString& name = createDescription.GetName();
         const ui32 shardsCount = Max(ui32(1), createDescription.GetColumnShardCount());
         auto opTxId = OperationId.GetTxId();
diff --git a/ydb/core/tx/schemeshard/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap.cpp
index d750c83e7e..0a5ca55d2b 100644
--- a/ydb/core/tx/schemeshard/ut_olap.cpp
+++ b/ydb/core/tx/schemeshard/ut_olap.cpp
@@ -211,6 +211,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
 
         TString tableSchema = R"(
             Name: "ColumnTable"
+            ColumnShardCount: 1
         )";
 
         TestCreateColumnTable(runtime, ++txId, "/MyRoot/OlapStore/MyDir", tableSchema);
@@ -223,6 +224,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
         // Missing column from schema preset
         TestCreateColumnTable(runtime, ++txId, "/MyRoot/OlapStore/MyDir", R"(
             Name: "ColumnTableMissingDataColumn"
+            ColumnShardCount: 1
             Schema {
                 Columns { Name: "timestamp" Type: "Timestamp" }
                 KeyColumnNames: "timestamp"
@@ -233,6 +235,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
         // Extra column not in schema preset
         TestCreateColumnTable(runtime, ++txId, "/MyRoot/OlapStore/MyDir", R"(
             Name: "ColumnTableExtraColumn"
+            ColumnShardCount: 1
             Schema {
                 Columns { Name: "timestamp" Type: "Timestamp" }
                 Columns { Name: "data" Type: "Utf8" }
@@ -245,6 +248,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
         // Different column order
         TestCreateColumnTable(runtime, ++txId, "/MyRoot/OlapStore/MyDir", R"(
             Name: "ColumnTableDifferentColumnOrder"
+            ColumnShardCount: 1
             Schema {
                 Columns { Name: "data" Type: "Utf8" }
                 Columns { Name: "timestamp" Type: "Timestamp" }
@@ -256,6 +260,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
         // Extra key column
         TestCreateColumnTable(runtime, ++txId, "/MyRoot/OlapStore/MyDir", R"(
             Name: "ColumnTableExtraKeyColumn"
+            ColumnShardCount: 1
             Schema {
                 Columns { Name: "timestamp" Type: "Timestamp" }
                 Columns { Name: "data" Type: "Utf8" }
@@ -268,6 +273,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
         // Unknown key column
         TestCreateColumnTable(runtime, ++txId, "/MyRoot/OlapStore/MyDir", R"(
             Name: "ColumnTableUnknownKeyColumn"
+            ColumnShardCount: 1
             Schema {
                 Columns { Name: "timestamp" Type: "Timestamp" }
                 Columns { Name: "data" Type: "Utf8" }
@@ -279,6 +285,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
         // Different data column type
         TestCreateColumnTable(runtime, ++txId, "/MyRoot/OlapStore/MyDir", R"(
             Name: "ColumnTableDataColumnType"
+            ColumnShardCount: 1
             Schema {
                 Columns { Name: "timestamp" Type: "Timestamp" }
                 Columns { Name: "data" Type: "String" }
@@ -290,6 +297,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
         // Repeating preset schema should succeed
         TestCreateColumnTable(runtime, ++txId, "/MyRoot/OlapStore/MyDir", R"(
             Name: "ColumnTableExplicitSchema"
+            ColumnShardCount: 1
             Schema {
                 Columns { Name: "timestamp" Type: "Timestamp" }
                 Columns { Name: "data" Type: "Utf8" }
@@ -302,6 +310,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
         // Creating table with directories should succeed
         TestCreateColumnTable(runtime, ++txId, "/MyRoot/OlapStore", R"(
             Name: "DirA/DirB/NestedTable"
+            ColumnShardCount: 1
         )");
         env.TestWaitNotification(runtime, txId);
 
@@ -312,6 +321,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
         // Additional storage tier in schema
         TestCreateColumnTable(runtime, ++txId, "/MyRoot/OlapStore/MyDir", R"(
             Name: "TableWithTiers"
+            ColumnShardCount: 1
             Schema {
                 Columns { Name: "timestamp" Type: "Timestamp" }
                 Columns { Name: "data" Type: "Utf8" }
@@ -333,6 +343,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
 
         TestCreateColumnTable(runtime, ++txId, "/MyRoot/OlapStore", R"(
             Name: "ColumnTable"
+            ColumnShardCount: 1
             SchemaPresetName: "default"
         )");
         env.TestWaitNotification(runtime, txId);
@@ -362,6 +373,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
 
         TString tableSchema = R"(
             Name: "ColumnTable"
+            ColumnShardCount: 1
         )";
 
         TestCreateColumnTable(runtime, ++txId, "/MyRoot/OlapStore/MyDir", tableSchema);
@@ -453,6 +465,67 @@ Y_UNIT_TEST_SUITE(TOlap) {
         TestLsPathId(runtime, 4, NLs::PathStringEqual(""));
     }
 
+    Y_UNIT_TEST(CreateDropStandaloneTableDefaultSharding) {
+        TTestBasicRuntime runtime;
+        TTestEnv env(runtime);
+        ui64 txId = 100;
+
+        TestMkDir(runtime, ++txId, "/MyRoot", "MyDir");
+        env.TestWaitNotification(runtime, txId);
+
+        TestLs(runtime, "/MyRoot/MyDir", false, NLs::PathExist);
+
+        TestCreateColumnTable(runtime, ++txId, "/MyRoot/MyDir", defaultTableSchema);
+        env.TestWaitNotification(runtime, txId);
+
+        TestLsPathId(runtime, 3, NLs::PathStringEqual("/MyRoot/MyDir/ColumnTable"));
+
+        TestDropColumnTable(runtime, ++txId, "/MyRoot/MyDir", "ColumnTable");
+        env.TestWaitNotification(runtime, txId);
+
+        TestLs(runtime, "/MyRoot/MyDir/ColumnTable", false, NLs::PathNotExist);
+        TestLsPathId(runtime, 3, NLs::PathStringEqual(""));
+
+        TString otherSchema = R"(
+            Name: "ColumnTable"
+            Schema {
+                Columns { Name: "timestamp" Type: "Timestamp" NotNull: true }
+                Columns { Name: "some" Type: "Uint64" NotNull: true }
+                Columns { Name: "data" Type: "Utf8" NotNull: true }
+                KeyColumnNames: "some"
+                KeyColumnNames: "data"
+            }
+        )";
+
+        TestCreateColumnTable(runtime, ++txId, "/MyRoot/MyDir", otherSchema);
+        env.TestWaitNotification(runtime, txId);
+
+        auto checkFn = [&](const NKikimrScheme::TEvDescribeSchemeResult& record) {
+            UNIT_ASSERT_VALUES_EQUAL(record.GetPath(), "/MyRoot/MyDir/ColumnTable");
+
+            auto& description = record.GetPathDescription().GetColumnTableDescription();
+            UNIT_ASSERT_VALUES_EQUAL(description.GetColumnShardCount(), 64);
+
+            auto& sharding = description.GetSharding();
+            UNIT_ASSERT_VALUES_EQUAL(sharding.ColumnShardsSize(), 64);
+            UNIT_ASSERT(sharding.HasHashSharding());
+            auto& hashSharding = sharding.GetHashSharding();
+            UNIT_ASSERT_VALUES_EQUAL(hashSharding.ColumnsSize(), 2);
+            UNIT_ASSERT_EQUAL(hashSharding.GetFunction(),
+                              NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_MODULO_N);
+            UNIT_ASSERT_VALUES_EQUAL(hashSharding.GetColumns()[0], "some");
+            UNIT_ASSERT_VALUES_EQUAL(hashSharding.GetColumns()[1], "data");
+        };
+
+        TestLsPathId(runtime, 4, checkFn);
+
+        TestDropColumnTable(runtime, ++txId, "/MyRoot/MyDir", "ColumnTable");
+        env.TestWaitNotification(runtime, txId);
+
+        TestLs(runtime, "/MyRoot/MyDir/ColumnTable", false, NLs::PathNotExist);
+        TestLsPathId(runtime, 4, NLs::PathStringEqual(""));
+    }
+
     Y_UNIT_TEST(CreateTableTtl) {
         TTestBasicRuntime runtime;
         TTestEnv env(runtime);
@@ -463,6 +536,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
 
         TString tableSchema1 = R"(
             Name: "Table1"
+            ColumnShardCount: 1
             TtlSettings {
                 Enabled { ColumnName: "timestamp" ExpireAfterSeconds: 300 }
             }
@@ -479,6 +553,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
 
         TString tableSchema2 = R"(
             Name: "Table2"
+            ColumnShardCount: 1
             TtlSettings {
                 Disabled {}
             }
@@ -495,6 +570,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
 
         TString tableSchema3 = R"(
             Name: "Table3"
+            ColumnShardCount: 1
             TtlSettings {
                 UseTiering : "Tiering1"
             }
@@ -511,6 +587,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
 
         TString tableSchema4 = R"(
             Name: "Table4"
+            ColumnShardCount: 1
             TtlSettings {
                 UseTiering : "Tiering1"
             }
@@ -532,6 +609,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
 
         TString tableSchemaX = R"(
             Name: "ColumnTable"
+            ColumnShardCount: 1
             TtlSettings {
                 Enabled {
                     ExpireAfterSeconds: 300
@@ -544,6 +622,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
 
         TString tableSchema = R"(
             Name: "ColumnTable"
+            ColumnShardCount: 1
             TtlSettings {
                 Enabled {
                     ColumnName: "timestamp"
@@ -596,6 +675,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
 
         TString tableSchema = R"(
             Name: "ColumnTable"
+            ColumnShardCount: 1
             TtlSettings {
                 Enabled {
                     ColumnName: "timestamp"
@@ -679,6 +759,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
 
         TString tableSchema = R"(
             Name: "ColumnTable"
+            ColumnShardCount: 1
         )";
 
         TestCreateColumnTable(runtime, ++txId, "/MyRoot/OlapStore", tableSchema);
diff --git a/ydb/core/tx/schemeshard/ut_olap_reboots.cpp b/ydb/core/tx/schemeshard/ut_olap_reboots.cpp
index 64bad70f3d..47ee8880c5 100644
--- a/ydb/core/tx/schemeshard/ut_olap_reboots.cpp
+++ b/ydb/core/tx/schemeshard/ut_olap_reboots.cpp
@@ -66,6 +66,7 @@ Y_UNIT_TEST_SUITE(TOlapReboots) {
 
             TestCreateColumnTable(runtime, ++t.TxId, "/MyRoot/OlapStore", R"(
                 Name: "ColumnTable"
+                ColumnShardCount: 1
             )");
             t.TestEnv->TestWaitNotification(runtime, t.TxId);
 
@@ -111,6 +112,7 @@ Y_UNIT_TEST_SUITE(TOlapReboots) {
 
             TestCreateColumnTable(runtime, ++t.TxId, "/MyRoot/OlapStore", R"(
                 Name: "ColumnTable"
+                ColumnShardCount: 1
             )");
             t.TestEnv->TestWaitNotification(runtime, t.TxId);
 
@@ -163,11 +165,13 @@ Y_UNIT_TEST_SUITE(TOlapReboots) {
             t.TestEnv->ReliablePropose(runtime,
                 CreateColumnTableRequest(t.TxId += 2, "/MyRoot/OlapStore", R"(
                     Name: "ColumnTable1"
+                    ColumnShardCount: 1
                 )"),
                 {NKikimrScheme::StatusAccepted, NKikimrScheme::StatusAlreadyExists, NKikimrScheme::StatusMultipleModifications});
             t.TestEnv->ReliablePropose(runtime,
                 CreateColumnTableRequest(t.TxId - 1, "/MyRoot/OlapStore", R"(
                     Name: "ColumnTable2"
+                    ColumnShardCount: 1
                 )"),
                 {NKikimrScheme::StatusAccepted, NKikimrScheme::StatusAlreadyExists, NKikimrScheme::StatusMultipleModifications});
             t.TestEnv->TestWaitNotification(runtime, {t.TxId - 1, t.TxId});
@@ -221,11 +225,13 @@ Y_UNIT_TEST_SUITE(TOlapReboots) {
 
                 TestCreateColumnTable(runtime, ++t.TxId, "/MyRoot/OlapStore", R"(
                     Name: "ColumnTable1"
+                    ColumnShardCount: 1
                 )");
                 t.TestEnv->TestWaitNotification(runtime, t.TxId);
 
                 TestCreateColumnTable(runtime, ++t.TxId, "/MyRoot/OlapStore", R"(
                     Name: "ColumnTable2"
+                    ColumnShardCount: 1
                 )");
                 t.TestEnv->TestWaitNotification(runtime, t.TxId);
             }
@@ -319,6 +325,7 @@ Y_UNIT_TEST_SUITE(TOlapReboots) {
 
                 TestCreateColumnTable(runtime, ++t.TxId, "/MyRoot/OlapStore", R"(
                     Name: "ColumnTable"
+                    ColumnShardCount: 1
                 )");
                 t.TestEnv->TestWaitNotification(runtime, t.TxId);
             }
@@ -354,6 +361,7 @@ Y_UNIT_TEST_SUITE(TOlapReboots) {
 
                 TestCreateColumnTable(runtime, ++t.TxId, "/MyRoot/OlapStore", R"(
                     Name: "ColumnTable"
+                    ColumnShardCount: 1
                     SchemaPresetName: "default"
                     TtlSettings {
                         Enabled {