Set default storage type as `column` (was `row`) and default datetime mode as `datetime32` (was `datetime64`) in `ydb workload * init` commands. (#21433)

13 files changed, 47 insertions, 31 deletions

diff --git a/ydb/apps/ydb/CHANGELOG.md b/ydb/apps/ydb/CHANGELOG.md
index c808d2a039b..b77cd5008c2 100644
--- a/ydb/apps/ydb/CHANGELOG.md
+++ b/ydb/apps/ydb/CHANGELOG.md
@@ -1,3 +1,4 @@
+* Set default storage type as `column` (was `row`) and default datetime mode as `datetime32` (was `datetime64`) in `ydb workload * init` commands.
 * Added ability of `ydb workload tpch` and `ydb workload tpcds` commands to use fraction `--scale` option.
 * Added `ydb workload tpcc check` subcommand, which checks TPC-C data consistency.
 
diff --git a/ydb/library/workload/benchmark_base/workload.cpp b/ydb/library/workload/benchmark_base/workload.cpp
index 12112dd5481..646ca180040 100644
--- a/ydb/library/workload/benchmark_base/workload.cpp
+++ b/ydb/library/workload/benchmark_base/workload.cpp
@@ -2,6 +2,7 @@
 #include <contrib/libs/fmt/include/fmt/format.h>
 #include <ydb/public/api/protos/ydb_formats.pb.h>
 #include <ydb/library/yaml_json/yaml_to_json.h>
+#include <library/cpp/colorizer/colors.h>
 #include <contrib/libs/yaml-cpp/include/yaml-cpp/node/parse.h>
 #include <util/string/cast.h>
 #include <util/system/spinlock.h>
@@ -54,9 +55,18 @@ ui32 TWorkloadGeneratorBase::GetDefaultPartitionsCount(const TString& /*tableNam
 void TWorkloadGeneratorBase::GenerateDDLForTable(IOutputStream& result, const NJson::TJsonValue& table, const NJson::TJsonValue& common, bool single) const {
     auto specialTypes = GetSpecialDataTypes();
     specialTypes["string_type"] = Params.GetStringType();
-    specialTypes["date_type"] = Params.GetDateType();
-    specialTypes["datetime_type"] = Params.GetDatetimeType();
-    specialTypes["timestamp_type"] = Params.GetTimestampType();
+    switch (Params.GetDatetimeTypes()) {
+    case TWorkloadBaseParams::EDatetimeTypes::DateTime32:
+        specialTypes["date_type"] = "Date";
+        specialTypes["datetime_type"] = "Datetime";
+        specialTypes["timestamp_type"] = "Timestamp";
+        break;
+    case TWorkloadBaseParams::EDatetimeTypes::DateTime64:
+        specialTypes["date_type"] = "Date32";
+        specialTypes["datetime_type"] = "Datetime64";
+        specialTypes["timestamp_type"] = "Timestamp64";
+        break;
+    }
 
     const auto& tableName = table["name"].GetString();
     const auto path = Params.GetFullTableName((single && Params.GetPath())? nullptr : tableName.c_str());
@@ -183,6 +193,7 @@ TBulkDataGeneratorList TWorkloadDataInitializerBase::GetBulkInitialData() {
 }
 
 void TWorkloadBaseParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandType commandType, int /*workloadType*/) {
+    NColorizer::TColors colors = NColorizer::AutoColors(Cout);
     switch (commandType) {
     default:
         break;
@@ -206,8 +217,10 @@ void TWorkloadBaseParams::ConfigureOpts(NLastGetopt::TOpts& opts, const ECommand
             .Optional()
             .StoreResult(&S3Endpoint);
         opts.AddLongOption("string", "Use String type in tables instead Utf8 one.").NoArgument().StoreValue(&StringType, "String");
-        opts.AddLongOption("datetime", "Use Date and Timestamp types in tables instead Date32 and Timestamp64 ones.").NoArgument()
-            .StoreValue(&DateType, "Date").StoreValue(&TimestampType, "Timestamp").StoreValue(&DatetimeType, "Datetime");
+        opts.AddLongOption("datetime-types", TStringBuilder() << "Datetime types to use. Available options:"
+            "\n  " << colors.BoldColor() << EDatetimeTypes::DateTime32 << colors.OldColor() << "\n    Date, Datetime and Timestamp"
+            "\n  " << colors.BoldColor() << EDatetimeTypes::DateTime64 << colors.OldColor() << "\n    Date32, Datetime64 and Timestamp64"
+            "\nDefault: " << colors.CyanColor() << "\"" << DatetimeTypes << "\"" << colors.OldColor() << ".").StoreResult(&DatetimeTypes);
         opts.AddLongOption("partition-size", "Maximum partition size in megabytes (AUTO_PARTITIONING_PARTITION_SIZE_MB) for row tables.")
             .DefaultValue(PartitionSizeMb).StoreResult(&PartitionSizeMb);
         break;
diff --git a/ydb/library/workload/benchmark_base/workload.h b/ydb/library/workload/benchmark_base/workload.h
index fef4dbf704f..551359508a5 100644
--- a/ydb/library/workload/benchmark_base/workload.h
+++ b/ydb/library/workload/benchmark_base/workload.h
@@ -16,24 +16,26 @@ public:
     enum class EStoreType {
         Row     /* "row"    */,
         Column  /* "column" */,
-        ExternalS3      /* "external-s3"     */
+        ExternalS3      /* "external-s3" */
     };
     enum class EQuerySyntax {
         YQL /* "yql" */,
         PG /* "pg"*/
     };
+    enum class EDatetimeTypes {
+        DateTime32 /* "dt32" */,
+        DateTime64 /* "dt64" */
+    };
     void ConfigureOpts(NLastGetopt::TOpts& opts, const ECommandType commandType, int workloadType) override;
     void Validate(const ECommandType commandType, int workloadType) override;
     TString GetFullTableName(const char* table) const;
     static TString GetTablePathQuote(EQuerySyntax syntax);
     YDB_ACCESSOR_DEF(TString, Path);
-    YDB_READONLY(EStoreType, StoreType, EStoreType::Row);
+    YDB_READONLY(EStoreType, StoreType, EStoreType::Column);
     YDB_READONLY_DEF(TString, S3Endpoint);
     YDB_READONLY_DEF(TString, S3Prefix);
     YDB_READONLY(TString, StringType, "Utf8");
-    YDB_READONLY(TString, DateType, "Date32");
-    YDB_READONLY(TString, DatetimeType, "Datetime64");
-    YDB_READONLY(TString, TimestampType, "Timestamp64");
+    YDB_READONLY(EDatetimeTypes, DatetimeTypes, EDatetimeTypes::DateTime32);
     YDB_READONLY(ui64, PartitionSizeMb, 2000);
     YDB_READONLY_PROTECT(bool, CheckCanonical, false);
 };
diff --git a/ydb/tests/compatibility/test_compatibility.py b/ydb/tests/compatibility/test_compatibility.py
index 23521c9561e..1b779b03def 100644
--- a/ydb/tests/compatibility/test_compatibility.py
+++ b/ydb/tests/compatibility/test_compatibility.py
@@ -89,9 +89,9 @@ class TestCompatibility(RestartToAnotherVersionFixture):
         assert self.execute_scan_query('select count(*) as row_count from `sample_table`')[0]['row_count'] == 500, 'Expected 500 rows: update 100-200 rows and added 300 rows'
 
     @pytest.mark.parametrize("store_type, date_args", [
-        pytest.param("row",    ["--datetime"], id="row"),
-        pytest.param("column", ["--datetime"], id="column"),
-        pytest.param("column", []            , id="column-date64")
+        pytest.param("row",    ["--datetime-types=dt32"], id="row"),
+        pytest.param("column", ["--datetime-types=dt32"], id="column"),
+        pytest.param("column", ["--datetime-types=dt64"], id="column-date64")
     ])
     def test_tpch1(self, store_type, date_args):
         result_json_path = os.path.join(yatest.common.test_output_path(), "result.json")
diff --git a/ydb/tests/compatibility/test_stress.py b/ydb/tests/compatibility/test_stress.py
index 303e5cefa8d..d24abcc66e5 100644
--- a/ydb/tests/compatibility/test_stress.py
+++ b/ydb/tests/compatibility/test_stress.py
@@ -180,9 +180,9 @@ class TestStress(MixedClusterFixture):
         yatest.common.execute(run_command, wait=True, stdout=self.output_f, stderr=self.output_f)
 
     @pytest.mark.parametrize("store_type, date_args", [
-        pytest.param("row",    ["--datetime"], id="row"),
-        pytest.param("column", ["--datetime"], id="column"),
-        pytest.param("column", []            , id="column-date64")
+        pytest.param("row",    ["--datetime-types=dt32"], id="row"),
+        pytest.param("column", ["--datetime-types=dt32"], id="column"),
+        pytest.param("column", ["--datetime-types=dt64"], id="column-date64")
     ])
     def test_tpch1(self, store_type, date_args):
         init_command = [
@@ -236,9 +236,9 @@ class TestStress(MixedClusterFixture):
 
     @pytest.mark.skip(reason="Not stabilized yet")
     @pytest.mark.parametrize("store_type, date_args", [
-        pytest.param("row",    ["--datetime"], id="row"),
-        pytest.param("column", ["--datetime"], id="column"),
-        pytest.param("column", []            , id="column-date64")
+        pytest.param("row",    ["--datetime-types=dt32"], id="row"),
+        pytest.param("column", ["--datetime-types=dt32"], id="column"),
+        pytest.param("column", ["--datetime-types=dt64"], id="column-date64")
     ])
     def test_tpcds1(self, store_type, date_args):
         init_command = [
diff --git a/ydb/tests/functional/benchmarks_init/test_init.py b/ydb/tests/functional/benchmarks_init/test_init.py
index 5f9137e4da8..071e414a7d2 100644
--- a/ydb/tests/functional/benchmarks_init/test_init.py
+++ b/ydb/tests/functional/benchmarks_init/test_init.py
@@ -30,7 +30,7 @@ class InitBase(object):
                 '--endpoint', 'grpc://localhost',
                 '--database', '/Root/db',
                 'workload', cls.workload, '-p', f'/Root/db/{cls.workload}/s{scale}',
-                'init', '--dry-run'
+                'init', '--dry-run', '--datetime-types=dt64'
             ]
             + [str(arg) for arg in args]
         ).stdout.decode('utf8')
diff --git a/ydb/tests/functional/clickbench/test.py b/ydb/tests/functional/clickbench/test.py
index 5ad951cdeb6..7aa36a0dec3 100644
--- a/ydb/tests/functional/clickbench/test.py
+++ b/ydb/tests/functional/clickbench/test.py
@@ -129,7 +129,7 @@ def save_canonical_data(data, fname):
 @pytest.mark.parametrize("store", ["row", "column"])
 def test_run_benchmark(store):
     path = "clickbench/benchmark/{}/hits".format(store)
-    ret = run_cli(["workload", "clickbench", "--path", path, "init", "--store", store, "--datetime"])
+    ret = run_cli(["workload", "clickbench", "--path", path, "init", "--store", store, "--datetime-types=dt32"])
     assert_that(ret.exit_code, is_(0))
 
     ret = run_cli(
@@ -149,7 +149,7 @@ def test_run_benchmark(store):
 @pytest.mark.parametrize("store", ["row", "column"])
 def test_run_determentistic(store):
     path = "clickbench/determentistic/{}/hits".format(store)
-    ret = run_cli(["workload", "clickbench", "--path", path, "init", "--store", store, "--datetime"])
+    ret = run_cli(["workload", "clickbench", "--path", path, "init", "--store", store, "--datetime-types=dt32"])
     assert_that(ret.exit_code, is_(0))
     ret = run_cli(
         [
@@ -179,7 +179,7 @@ def test_run_determentistic(store):
 @pytest.mark.parametrize("store", ["row", "column"])
 def test_plans(store):
     ret = run_cli(
-        ["workload", "clickbench", "--path", "clickbench/plans/{}/hits".format(store), "init", "--store", store, "--datetime"]
+        ["workload", "clickbench", "--path", "clickbench/plans/{}/hits".format(store), "init", "--store", store, "--datetime-types=dt32"]
     )
     assert_that(ret.exit_code, is_(0))
 
diff --git a/ydb/tests/functional/tpc/large/test_tpcds.py b/ydb/tests/functional/tpc/large/test_tpcds.py
index be22a396e33..6966aac0635 100644
--- a/ydb/tests/functional/tpc/large/test_tpcds.py
+++ b/ydb/tests/functional/tpc/large/test_tpcds.py
@@ -12,7 +12,7 @@ class TestTpcdsS1(tpcds.TestTpcds1, FunctionalTestBase):
     @classmethod
     def setup_class(cls) -> None:
         cls.setup_cluster(memory_controller_config=cls.memory_controller_config)
-        cls.run_cli(['workload', 'tpcds', '-p', f'olap_yatests/{cls._get_path()}', 'init', '--store=column'])
+        cls.run_cli(['workload', 'tpcds', '-p', f'olap_yatests/{cls._get_path()}', 'init', '--store=column', '--datetime-types=dt64'])
         cls.run_cli(['workload', 'tpcds', '-p', f'olap_yatests/{cls._get_path()}', 'import', 'generator', f'--scale={cls.scale}'])
         super().setup_class()
 
diff --git a/ydb/tests/functional/tpc/large/test_tpch_spilling.py b/ydb/tests/functional/tpc/large/test_tpch_spilling.py
index 1016ef5693d..662c2e1107e 100644
--- a/ydb/tests/functional/tpc/large/test_tpch_spilling.py
+++ b/ydb/tests/functional/tpc/large/test_tpch_spilling.py
@@ -31,7 +31,7 @@ class TestTpchSpillingS10(tpch.TestTpch10, FunctionalTestBase):
     @classmethod
     def setup_class(cls) -> None:
         cls.setup_cluster(table_service_config=cls.table_service_config, memory_controller_config=cls.memory_controller_config)
-        cls.run_cli(['workload', 'tpch', '-p', 'olap_yatests/tpch/s10', 'init', '--store=column'])
+        cls.run_cli(['workload', 'tpch', '-p', 'olap_yatests/tpch/s10', 'init', '--store=column', '--datetime-types=dt64'])
         cls.run_cli(['workload', 'tpch', '-p', 'olap_yatests/tpch/s10', 'import', 'generator', '--scale=10'])
 
         tpch.TestTpch10.setup_class()
diff --git a/ydb/tests/functional/tpc/medium/test_clickbench.py b/ydb/tests/functional/tpc/medium/test_clickbench.py
index 867087b6f03..b1c2ef58c0e 100644
--- a/ydb/tests/functional/tpc/medium/test_clickbench.py
+++ b/ydb/tests/functional/tpc/medium/test_clickbench.py
@@ -10,7 +10,7 @@ class TestClickbench(clickbench.TestClickbench, FunctionalTestBase):
     @classmethod
     def setup_class(cls) -> None:
         cls.setup_cluster()
-        cls.run_cli(['workload', 'clickbench', '-p', 'olap_yatests/clickbench/hits', 'init', '--store=column'])
+        cls.run_cli(['workload', 'clickbench', '-p', 'olap_yatests/clickbench/hits', 'init', '--store=column', '--datetime-types=dt64'])
         cls.run_cli(['workload', 'clickbench', '-p', 'olap_yatests/clickbench/hits', 'import', 'files', '--input', yatest.common.source_path("ydb/tests/functional/clickbench/data/hits.csv")])
         super().setup_class()
 
@@ -22,7 +22,7 @@ class TestClickbenchParallel(clickbench.TestClickbenchParallel8, FunctionalTestB
     @classmethod
     def setup_class(cls) -> None:
         cls.setup_cluster()
-        cls.run_cli(['workload', 'clickbench', '-p', 'olap_yatests/clickbench/hits', 'init', '--store=column'])
+        cls.run_cli(['workload', 'clickbench', '-p', 'olap_yatests/clickbench/hits', 'init', '--store=column', '--datetime-types=dt64'])
         cls.run_cli(['workload', 'clickbench', '-p', 'olap_yatests/clickbench/hits', 'import', 'files', '--input', yatest.common.source_path("ydb/tests/functional/clickbench/data/hits.csv")])
         super().setup_class()
 
@@ -34,6 +34,6 @@ class TestClickbenchPg(clickbench.TestClickbenchPg, FunctionalTestBase):
     @classmethod
     def setup_class(cls) -> None:
         cls.setup_cluster()
-        cls.run_cli(['workload', 'clickbench', '-p', 'olap_yatests/clickbench/hits', 'init', '--store=column'])
+        cls.run_cli(['workload', 'clickbench', '-p', 'olap_yatests/clickbench/hits', 'init', '--store=column', '--datetime-types=dt64'])
         cls.run_cli(['workload', 'clickbench', '-p', 'olap_yatests/clickbench/hits', 'import', 'files', '--input', yatest.common.source_path("ydb/tests/functional/clickbench/data/hits.csv")])
         super().setup_class()
diff --git a/ydb/tests/functional/tpc/medium/tpch/test_s1.py b/ydb/tests/functional/tpc/medium/tpch/test_s1.py
index 6189fbf1cfb..ffbd51568da 100644
--- a/ydb/tests/functional/tpc/medium/tpch/test_s1.py
+++ b/ydb/tests/functional/tpc/medium/tpch/test_s1.py
@@ -14,6 +14,6 @@ class TestTpchS1(tpch.TestTpch1, FunctionalTestBase):
     @classmethod
     def setup_class(cls) -> None:
         cls.setup_cluster()
-        cls.run_cli(['workload', 'tpch', '-p', f'olap_yatests/{cls._get_path()}', 'init', '--store=column'] + cls.addition_init_params())
+        cls.run_cli(['workload', 'tpch', '-p', f'olap_yatests/{cls._get_path()}', 'init', '--store=column', '--datetime-types=dt64'] + cls.addition_init_params())
         cls.run_cli(['workload', 'tpch', '-p', f'olap_yatests/{cls._get_path()}', 'import', 'generator', f'--scale={cls.scale}'])
         super().setup_class()
diff --git a/ydb/tests/olap/s3_import/test_tpch_import.py b/ydb/tests/olap/s3_import/test_tpch_import.py
index 18a17b298f3..fb383b20390 100644
--- a/ydb/tests/olap/s3_import/test_tpch_import.py
+++ b/ydb/tests/olap/s3_import/test_tpch_import.py
@@ -90,7 +90,7 @@ class TestS3TpchImport(S3ImportTestBase):
         """)
 
         logger.info("Creating tpc-h tables...")
-        self.ydb_client.run_cli_comand(["workload", "tpch", "init", "--datetime", "--store", "column"])
+        self.ydb_client.run_cli_comand(["workload", "tpch", "init", "--datetime-types=dt32", "--store", "column"])
         self.ydb_client.run_cli_comand(["workload", "tpch", "import", "generator", "--scale", "1"])
 
         logger.info("Exporting into s3...")
diff --git a/ydb/tests/sql/lib/test_base.py b/ydb/tests/sql/lib/test_base.py
index 858ce30877c..b0524243e17 100644
--- a/ydb/tests/sql/lib/test_base.py
+++ b/ydb/tests/sql/lib/test_base.py
@@ -109,7 +109,7 @@ class TpchTestBaseH1(TestBase):
         cls.teardown_tpch()
 
     def setup_tpch(cls):
-        cls.run_cli(['workload', 'tpch', '-p', cls.tpch_default_path()+'/', 'init', '--store=column', '--datetime'])
+        cls.run_cli(['workload', 'tpch', '-p', cls.tpch_default_path()+'/', 'init', '--store=column', '--datetime-types=dt32'])
         cls.run_cli(['workload', 'tpch', '-p', cls.tpch_default_path()+'/', 'import', 'generator', '--scale=1'])
 
     def teardown_tpch(cls):