diff options
| author | xyliganSereja <[email protected]> | 2026-07-22 13:16:04 +0300 |
|---|---|---|
| committer | GitHub <[email protected]> | 2026-07-22 13:16:04 +0300 |
| commit | 39d9cfbbfc88f62c145c83919e9233dbc2e17724 (patch) | |
| tree | 777eee72003a15a0852c5a3d59751dd1d38900f7 | |
| parent | 7f8ff20584d754289a7864dcb64d3c9d98c5f30a (diff) | |
simplify tests (#47368)
Co-authored-by: Matveev Sergei <[email protected]>
| -rw-r--r-- | ydb/core/kqp/ut/olap/types/bool_ut.cpp | 648 | ||||
| -rw-r--r-- | ydb/core/kqp/ut/olap/types/column_type_scenarios.h | 165 | ||||
| -rw-r--r-- | ydb/core/kqp/ut/olap/types/column_type_test_base.h | 299 | ||||
| -rw-r--r-- | ydb/core/kqp/ut/olap/types/column_type_test_enums.h (renamed from ydb/core/kqp/ut/olap/types/bool_test_enums.h) | 0 | ||||
| -rw-r--r-- | ydb/core/kqp/ut/olap/types/decimal_ut.cpp | 789 | ||||
| -rw-r--r-- | ydb/core/kqp/ut/olap/types/dynumber_ut.cpp | 792 | ||||
| -rw-r--r-- | ydb/core/kqp/ut/olap/types/interval_ut.cpp | 693 | ||||
| -rw-r--r-- | ydb/core/kqp/ut/olap/types/uuid_ut.cpp | 771 | ||||
| -rw-r--r-- | ydb/core/kqp/ut/olap/types/ya.make | 2 |
9 files changed, 1782 insertions, 2377 deletions
diff --git a/ydb/core/kqp/ut/olap/types/bool_ut.cpp b/ydb/core/kqp/ut/olap/types/bool_ut.cpp index 411f6a0603c..9b2dda5f0ca 100644 --- a/ydb/core/kqp/ut/olap/types/bool_ut.cpp +++ b/ydb/core/kqp/ut/olap/types/bool_ut.cpp @@ -1,26 +1,10 @@ -#include "bool_test_enums.h" +#include "column_type_scenarios.h" +#include "column_type_test_base.h" -#include <ydb/core/formats/arrow/arrow_helpers.h> -#include <ydb/core/kqp/ut/common/columnshard.h> -#include <ydb/core/kqp/ut/common/kqp_ut_common.h> -#include <ydb/core/testlib/common_helper.h> -#include <ydb/core/testlib/cs_helper.h> -#include <ydb/core/tx/columnshard/hooks/testing/controller.h> #include <ydb/core/tx/columnshard/test_helper/test_combinator.h> -#include <ydb/core/tx/tx_proxy/proxy.h> - -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_replication.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/client.h> - #include <ydb/library/actors/core/log.h> -#include <library/cpp/threading/local_executor/local_executor.h> -#include <util/generic/serialized_enum.h> -#include <util/string/printf.h> #include <yql/essentials/types/binary_json/write.h> #include <yql/essentials/types/uuid/uuid.h> -#include <ydb/core/kqp/ut/common/arrow_builders.h> namespace NKikimr { namespace NKqp { @@ -30,114 +14,55 @@ using namespace NYdb::NTable; Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { namespace { - struct TRow { - i32 Id; - i64 IntVal; - std::optional<bool> B; - }; - TKikimrSettings CreateKikimrSettingsWithBoolSupport() { - NKikimrConfig::TFeatureFlags featureFlags; - featureFlags.SetEnableColumnshardBool(true); - return TKikimrSettings().SetWithSampleTables(false).SetFeatureFlags(featureFlags); - } + struct TBoolTraits { + using TValue = bool; + static constexpr const char* ColumnName = "b"; + static constexpr const char* SqlTypeName = "Bool"; - void CreateDataShardTable(TTestHelper& helper, const TString& name) { - auto& session = helper.GetSession(); - auto res = session - .ExecuteSchemeQuery(TStringBuilder() << R"( - CREATE TABLE `)" << name << R"(` ( - id Int32 NOT NULL, - int Int64, - b Bool, - PRIMARY KEY (id) - ); - )") - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), NYdb::EStatus::SUCCESS); - } + static TKikimrSettings CreateSettings() { + return CreateColumnshardSettings([](auto& f) { f.SetEnableColumnshardBool(true); }); + } - void CreateDataShardTableWithSecondColumn(TTestHelper& helper, const TString& name, const TString& secondName) { - auto& session = helper.GetSession(); - auto res = session - .ExecuteSchemeQuery(TStringBuilder() << R"( - CREATE TABLE `)" << name << R"(` ( - id Int32 NOT NULL, - )" << secondName << R"( Int64, - b Bool, - PRIMARY KEY (id) - ); - )") - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), NYdb::EStatus::SUCCESS); - } + static auto GetTypeId() { return NScheme::NTypeIds::Bool; } - void BulkUpsertRowTableYdbValueWithColumnName( - TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& columnName) { - TValueBuilder builder; - builder.BeginList(); - for (auto&& r : rows) { - builder.AddListItem().BeginStruct().AddMember("id").Int32(r.Id).AddMember(columnName).Int64(r.IntVal).AddMember("b"); - if (r.B.has_value()) { - builder.BeginOptional().Bool(*r.B).EndOptional(); + static void AppendYdbValue(TValueBuilder& builder, const std::optional<bool>& val) { + if (val.has_value()) { + builder.BeginOptional().Bool(*val).EndOptional(); } else { builder.EmptyOptional(EPrimitiveType::Bool); } - - builder.EndStruct(); } - builder.EndList(); - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, builder.Build()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void BulkUpsertRowTableYdbValue(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - BulkUpsertRowTableYdbValueWithColumnName(helper, name, rows, "int"); - } - - void BulkUpsertRowTableYdbValueWithSecondColumn( - TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& secondName) { - TValueBuilder builder; - builder.BeginList(); - for (auto&& r : rows) { - builder.AddListItem().BeginStruct().AddMember("id").Int32(r.Id).AddMember(secondName).Int64(r.IntVal).AddMember("b"); - if (r.B.has_value()) { - builder.BeginOptional().Bool(*r.B).EndOptional(); - } else { - builder.EmptyOptional(EPrimitiveType::Bool); + static void AppendCsvValue(TStringBuilder& builder, const std::optional<bool>& val) { + if (val.has_value()) { + builder << (*val ? "true" : "false"); } - - builder.EndStruct(); } - builder.EndList(); - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, builder.Build()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void BulkUpsertRowTableCSV(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - TStringBuilder builder; - for (auto&& r : rows) { - builder << r.Id << "," << r.IntVal << ","; - if (r.B.has_value()) { - builder << (*r.B ? "true" : "false"); + static std::shared_ptr<arrow::Array> MakeArrowArray(const TVector<TTypedRow<bool>>& rows) { + using namespace NKikimr::NKqp::NTestArrow; + std::vector<std::optional<bool>> bs; + bs.reserve(rows.size()); + for (auto&& r : rows) { + bs.push_back(r.TypedVal); } - builder << '\n'; + return MakeBoolArrayAsUInt8Nullable(bs); } - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, EDataFormat::CSV, builder).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } + static std::shared_ptr<arrow::DataType> ArrowType() { return arrow::uint8(); } + }; + + COLUMN_TYPE_TEST_USING(TBoolTraits); void BulkUpsertRowTableCSVWithFormat( TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& trueValue, const TString& falseValue) { TStringBuilder builder; for (auto&& r : rows) { builder << r.Id << "," << r.IntVal << ","; - if (r.B.has_value()) { - builder << (*r.B ? trueValue : falseValue); + if (r.TypedVal.has_value()) { + builder << (*r.TypedVal ? trueValue : falseValue); } builder << '\n'; @@ -147,148 +72,6 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); } - std::shared_ptr<arrow::RecordBatch> MakeArrowBatchWithColumnName(const TVector<TRow>& rows, const TString& columnName) { - using namespace NKikimr::NKqp::NTestArrow; - std::vector<int32_t> ids; - std::vector<int64_t> vals; - std::vector<std::optional<bool>> bs; - ids.reserve(rows.size()); - vals.reserve(rows.size()); - bs.reserve(rows.size()); - for (auto&& r : rows) { - ids.push_back(r.Id); - vals.push_back(r.IntVal); - bs.push_back(r.B); - } - - auto idArr = MakeInt32Array(ids); - auto intArr = MakeInt64Array(vals); - auto boolArr = MakeBoolArrayAsUInt8Nullable(bs); - auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), /*nullable*/ false), - arrow::field(columnName, arrow::int64()), - arrow::field("b", arrow::uint8()) - }); - - return MakeBatch({ schema->field(0), schema->field(1), schema->field(2) }, { idArr, intArr, boolArr }); - } - - std::shared_ptr<arrow::RecordBatch> MakeArrowBatch(const TVector<TRow>& rows) { - return MakeArrowBatchWithColumnName(rows, "int"); - } - - std::shared_ptr<arrow::RecordBatch> MakeArrowBatchWithSecondColumn(const TVector<TRow>& rows, const TString& secondName) { - using namespace NKikimr::NKqp::NTestArrow; - std::vector<int32_t> ids; - std::vector<int64_t> seconds; - std::vector<std::optional<bool>> bs; - ids.reserve(rows.size()); - seconds.reserve(rows.size()); - bs.reserve(rows.size()); - for (auto&& r : rows) { - ids.push_back(r.Id); - seconds.push_back(r.IntVal); - bs.push_back(r.B); - } - - auto idArr = MakeInt32Array(ids); - auto secondArr = MakeInt64Array(seconds); - auto boolArr = MakeBoolArrayAsUInt8Nullable(bs); - auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), /*nullable*/ false), - arrow::field(secondName, arrow::int64()), - arrow::field("b", arrow::uint8()) - }); - - return MakeBatch({ schema->field(0), schema->field(1), schema->field(2) }, { idArr, secondArr, boolArr }); - } - - void BulkUpsertRowTableArrow(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - auto batch = MakeArrowBatch(rows); - TString strBatch = NArrow::SerializeBatchNoCompression(batch); - TString strSchema = NArrow::SerializeSchema(*batch->schema()); - auto result = - helper.GetKikimr().GetTableClient().BulkUpsert(name, NYdb::NTable::EDataFormat::ApacheArrow, strBatch, strSchema).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void BulkUpsertRowTableArrowWithSecondColumn( - TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& secondName) { - auto batch = MakeArrowBatchWithSecondColumn(rows, secondName); - TString strBatch = NArrow::SerializeBatchNoCompression(batch); - TString strSchema = NArrow::SerializeSchema(*batch->schema()); - auto result = - helper.GetKikimr().GetTableClient().BulkUpsert(name, NYdb::NTable::EDataFormat::ApacheArrow, strBatch, strSchema).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void LoadData(TTestHelper& helper, ETableKind table, ELoadKind load, const TString& name, const TVector<TRow>& rows, - TTestHelper::TColumnTable* col = nullptr, const TVector<TTestHelper::TColumnSchema>* schema = nullptr) { - switch (table) { - case ETableKind::COLUMNSHARD: { - Y_ABORT_UNLESS(col && schema); - if (load == ELoadKind::ARROW) { - TString columnName = "int"; - if (schema->size() >= 2) { - columnName = (*schema)[1].GetName(); - } - - auto batch = MakeArrowBatchWithColumnName(rows, columnName); - helper.BulkUpsert(*col, batch); - } else if (load == ELoadKind::YDB_VALUE) { - TString columnName = "int"; - if (schema->size() >= 2) { - columnName = (*schema)[1].GetName(); - } - - BulkUpsertRowTableYdbValueWithColumnName(helper, name, rows, columnName); - } else { - BulkUpsertRowTableCSV(helper, name, rows); - } - - break; - } - case ETableKind::DATASHARD: { - if (load == ELoadKind::ARROW) { - BulkUpsertRowTableArrow(helper, name, rows); - } else if (load == ELoadKind::YDB_VALUE) { - BulkUpsertRowTableYdbValue(helper, name, rows); - } else { - BulkUpsertRowTableCSV(helper, name, rows); - } - - break; - } - } - } - - void CheckOrExec(TTestHelper& helper, const TString& query, const TString& expected, EQueryMode scanMode) { - if (scanMode == EQueryMode::SCAN_QUERY) { - helper.ReadData(query, expected); - } else { - helper.ReadDataExecQuery(query, expected); - } - } - - void PrepareBase(TTestHelper& helper, ETableKind tableKind, const TString& tableName, TTestHelper::TColumnTable* colTableOut, - TVector<TTestHelper::TColumnSchema>* schemaOut) { - if (tableKind == ETableKind::COLUMNSHARD) { - TVector<TTestHelper::TColumnSchema> schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("b").SetType(NScheme::NTypeIds::Bool), - }; - - *schemaOut = schema; - TTestHelper::TColumnTable col; - col.SetName(tableName).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema); - helper.CreateTable(col); - *colTableOut = col; - } else { - CreateDataShardTable(helper, tableName); - } - } - bool TryPrepareBase(TTestHelper& helper, ETableKind tableKind, const TString& tableName, TTestHelper::TColumnTable* colTableOut, TVector<TTestHelper::TColumnSchema>* schemaOut) { if (tableKind == ETableKind::COLUMNSHARD) { @@ -309,224 +92,118 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { return false; } } else { - CreateDataShardTable(helper, tableName); + Base::CreateDataShardTable(helper, tableName); return true; } } - } // namespace - - class TBoolTestCase { - public: - TBoolTestCase() - : TestHelper(CreateKikimrSettingsWithBoolSupport()) { - } - - TTestHelper::TUpdatesBuilder Inserter() { - return TTestHelper::TUpdatesBuilder(TestTable.GetArrowSchema(Schema)); - } - - void Upsert(TTestHelper::TUpdatesBuilder& inserter) { - TestHelper.BulkUpsert(TestTable, inserter); - } - - void CheckQuery(const TString& query, const TString& expected, EQueryMode mode = EQueryMode::SCAN_QUERY) const { - switch (mode) { - case EQueryMode::SCAN_QUERY: - TestHelper.ReadData(query, expected); - break; - case EQueryMode::EXECUTE_QUERY: { - TestHelper.ExecuteQuery(query); - break; - } - } - } - - void ExecuteDataQuery(const TString& query) const { - TestHelper.ExecuteQuery(query); - } - void PrepareTable1() { - Schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("b").SetType(NScheme::NTypeIds::Bool), - }; - - TestTable.SetName("/Root/Table1").SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(Schema); - TestHelper.CreateTable(TestTable); - - { - TTestHelper::TUpdatesBuilder inserter = Inserter(); - inserter.AddRow().Add(1).Add(4).Add(true); - inserter.AddRow().Add(2).Add(3).Add(false); - Upsert(inserter); - } + TScenario<bool> FilterEqualScenario() { + return { + "/Root/Table1", + { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true } }, { - TTestHelper::TUpdatesBuilder inserter = Inserter(); - inserter.AddRow().Add(4).Add(1).Add(true); - inserter.AddRow().Add(3).Add(2).Add(true); - - Upsert(inserter); - } - } - - void PrepareTable2() { - Schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("table1_id").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("b").SetType(NScheme::NTypeIds::Bool), - }; - - TestTable.SetName("/Root/Table2").SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(Schema); - TestHelper.CreateTable(TestTable); + { "SELECT * FROM `/Root/Table1` WHERE b == true", "[[[%true];1;[4]];[[%true];3;[2]];[[%true];4;[1]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE b != true order by id", "[[[%false];2;[3]]]" }, + }, + }; + } + TScenario<bool> FilterNullsScenario() { + return { + "/Root/Table1", + { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true }, { 5, 5, std::nullopt }, { 6, 6, std::nullopt } }, { - TTestHelper::TUpdatesBuilder inserter = Inserter(); - inserter.AddRow().Add(1).Add(1).Add(true); - inserter.AddRow().Add(2).Add(1).Add(false); - inserter.AddRow().Add(3).Add(2).Add(true); - inserter.AddRow().Add(4).Add(2).Add(false); - Upsert(inserter); - } - } - - void PrepareTable3() { - Schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("b").SetType(NScheme::NTypeIds::Bool).SetNullable(false), - }; - - TestTable.SetName("/Root/Table3").SetPrimaryKey({ "b" }).SetSchema(Schema); - TestHelper.CreateTable(TestTable); + { "SELECT * FROM `/Root/Table1` WHERE b is NULL order by id", "[[#;5;[5]];[#;6;[6]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE b is not NULL order by id", + "[[[%true];1;[4]];[[%false];2;[3]];[[%true];3;[2]];[[%true];4;[1]]]" }, + }, + }; + } + TScenario<bool> FilterCompareScenario() { + return { + "/Root/Table1", + { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true } }, { - TTestHelper::TUpdatesBuilder inserter = Inserter(); - inserter.AddRow().Add(1).Add(true); - inserter.AddRow().Add(2).Add(false); - Upsert(inserter); - } - } - - private: - TTestHelper TestHelper; - TVector<TTestHelper::TColumnSchema> Schema; - TTestHelper::TColumnTable TestTable; - }; - - Y_UNIT_TEST(TestSimpleQueries, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { { 1, 4, true }, { 2, 3, false }, { 4, 1, true }, { 3, 2, true } }, &col, &schema); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=1", "[[[%true];1;[4]]]", Scan); - CheckOrExec( - helper, "SELECT * FROM `/Root/Table1` order by id", "[[[%true];1;[4]];[[%false];2;[3]];[[%true];3;[2]];[[%true];4;[1]]]", Scan); + { "SELECT * FROM `/Root/Table1` WHERE b < true order by id", "[[[%false];2;[3]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE b > false order by id", "[[[%true];1;[4]];[[%true];3;[2]];[[%true];4;[1]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE b <= true order by id", + "[[[%true];1;[4]];[[%false];2;[3]];[[%true];3;[2]];[[%true];4;[1]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE b >= true order by id", "[[[%true];1;[4]];[[%true];3;[2]];[[%true];4;[1]]]" }, + }, + }; } - Y_UNIT_TEST(TestFilterEqual, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { { 1, 4, true }, { 2, 3, false }, { 4, 1, true }, { 3, 2, true } }, &col, &schema); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE b == true", "[[[%true];1;[4]];[[%true];3;[2]];[[%true];4;[1]]]", Scan); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE b != true order by id", "[[[%false];2;[3]]]", Scan); + TScenario<bool> OrderByScenario() { + return { + "/Root/Table1", + { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true } }, + { + { "SELECT * FROM `/Root/Table1` order by b, id", + "[[[%false];2;[3]];[[%true];1;[4]];[[%true];3;[2]];[[%true];4;[1]]]" }, + }, + }; } - Y_UNIT_TEST(TestFilterNulls, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true }, { 5, 5, std::nullopt }, { 6, 6, std::nullopt } }, &col, &schema); - const TString expectedNulls = "[[#;5;[5]];[#;6;[6]]]"; - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE b is NULL order by id", expectedNulls, Scan); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE b is not NULL order by id", - "[[[%true];1;[4]];[[%false];2;[3]];[[%true];3;[2]];[[%true];4;[1]]]", Scan); + TScenario<bool> GroupByScenario() { + return { + "/Root/Table1", + { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true }, { 5, 12, true }, { 6, 30, false } }, + { + { "SELECT b, count(*) FROM `/Root/Table1` group by b order by b", "[[[%false];2u];[[%true];4u]]" }, + }, + }; } - Y_UNIT_TEST(TestFilterCompare, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true } }, &col, &schema); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE b < true order by id", "[[[%false];2;[3]]]", Scan); - CheckOrExec( - helper, "SELECT * FROM `/Root/Table1` WHERE b > false order by id", "[[[%true];1;[4]];[[%true];3;[2]];[[%true];4;[1]]]", Scan); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE b <= true order by id", - "[[[%true];1;[4]];[[%false];2;[3]];[[%true];3;[2]];[[%true];4;[1]]]", Scan); - CheckOrExec( - helper, "SELECT * FROM `/Root/Table1` WHERE b >= true order by id", "[[[%true];1;[4]];[[%true];3;[2]];[[%true];4;[1]]]", Scan); + TScenario<bool> AggregationScenario() { + return { + "/Root/Table1", + { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true } }, + { + { "SELECT min(b) FROM `/Root/Table1`", "[[[%false]]]" }, + { "SELECT max(b) FROM `/Root/Table1`", "[[[%true]]]" }, + }, + }; } - Y_UNIT_TEST(TestOrderByBool, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true } }, &col, &schema); - CheckOrExec( - helper, "SELECT * FROM `/Root/Table1` order by b, id", "[[[%false];2;[3]];[[%true];1;[4]];[[%true];3;[2]];[[%true];4;[1]]]", Scan); + TJoinScenario<bool> JoinByBoolScenario() { + return { + "/Root/Table1", + "/Root/Table2", + { { 2, 3, true }, { 4, 1, true } }, + { { 2, 2, false }, { 4, 4, false }, { 1, 1, true }, { 3, 3, true } }, + { + { "SELECT t1.id, t2.id, t1.b FROM `/Root/Table1` as t1 join `/Root/Table2` as t2 on t1.b = t2.b order by t1.id, t2.id, t1.b", + R"([[2;1;[%true]];[2;3;[%true]];[4;1;[%true]];[4;3;[%true]]])" }, + }, + }; } - Y_UNIT_TEST(TestGroupByBool, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true }, { 5, 12, true }, { 6, 30, false } }, &col, &schema); - CheckOrExec(helper, "SELECT b, count(*) FROM `/Root/Table1` group by b order by b", "[[[%false];2u];[[%true];4u]]", Scan); - } + } // namespace - Y_UNIT_TEST(TestAggregation, EQueryMode, ETableKind, ELoadKind) { + Y_UNIT_TEST(TestSimpleQueries, EQueryMode, ETableKind, ELoadKind) { const auto Scan = Arg<0>(); const auto Table = Arg<1>(); const auto Load = Arg<2>(); const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); TTestHelper::TColumnTable col; TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { { 1, 4, true }, { 2, 3, false }, { 3, 2, true }, { 4, 1, true } }, &col, &schema); - CheckOrExec(helper, "SELECT min(b) FROM `/Root/Table1`", "[[[%false]]]", Scan); - CheckOrExec(helper, "SELECT max(b) FROM `/Root/Table1`", "[[[%true]]]", Scan); + Base::PrepareBase(helper, Table, tableName, &col, &schema); + Base::LoadData(helper, Table, Load, tableName, { { 1, 4, true }, { 2, 3, false }, { 4, 1, true }, { 3, 2, true } }, &col, &schema); + CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=1", "[[[%true];1;[4]]]", Scan); + CheckOrExec( + helper, "SELECT * FROM `/Root/Table1` order by id", "[[[%true];1;[4]];[[%false];2;[3]];[[%true];3;[2]];[[%true];4;[1]]]", Scan); } + Y_UNIT_TEST_SCENARIO(TestFilterEqual, FilterEqualScenario); + Y_UNIT_TEST_SCENARIO(TestFilterNulls, FilterNullsScenario); + Y_UNIT_TEST_SCENARIO(TestFilterCompare, FilterCompareScenario); + Y_UNIT_TEST_SCENARIO(TestOrderByBool, OrderByScenario); + Y_UNIT_TEST_SCENARIO(TestGroupByBool, GroupByScenario); + Y_UNIT_TEST_SCENARIO(TestAggregation, AggregationScenario); + Y_UNIT_TEST(TestJoinById, EQueryMode, ETableKind, ELoadKind) { const auto Scan = Arg<0>(); const auto Table = Arg<1>(); @@ -534,7 +211,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { const TString t1 = "/Root/Table1"; const TString t2 = "/Root/Table2"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); TTestHelper::TColumnTable col1, col2; TVector<TTestHelper::TColumnSchema> s1, s2; if (Table == ETableKind::COLUMNSHARD) { @@ -555,19 +232,19 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { col2.SetName(t2).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s2); helper.CreateTable(col2); } else { - CreateDataShardTable(helper, t1); - CreateDataShardTableWithSecondColumn(helper, t2, "table1_id"); + Base::CreateDataShardTable(helper, t1); + Base::CreateDataShardTableWithSecondColumn(helper, t2, "table1_id"); } - LoadData(helper, Table, Load, t1, { { 1, 4, true }, { 2, 3, true } }, &col1, &s1); + Base::LoadData(helper, Table, Load, t1, { { 1, 4, true }, { 2, 3, true } }, &col1, &s1); if (Table == ETableKind::COLUMNSHARD) { - LoadData(helper, Table, Load, t2, { { 1, 1, true }, { 2, 1, false }, { 3, 2, true }, { 4, 2, false } }, &col2, &s2); + Base::LoadData(helper, Table, Load, t2, { { 1, 1, true }, { 2, 1, false }, { 3, 2, true }, { 4, 2, false } }, &col2, &s2); } else { if (Load == ELoadKind::ARROW) { - BulkUpsertRowTableArrowWithSecondColumn( + Base::BulkUpsertRowTableArrowWithSecondColumn( helper, t2, { { 1, 1, true }, { 2, 1, false }, { 3, 2, true }, { 4, 2, false } }, "table1_id"); } else if (Load == ELoadKind::YDB_VALUE) { - BulkUpsertRowTableYdbValueWithSecondColumn( + Base::BulkUpsertRowTableYdbValueWithSecondColumn( helper, t2, { { 1, 1, true }, { 2, 1, false }, { 3, 2, true }, { 4, 2, false } }, "table1_id"); } else { TStringBuilder csv; @@ -582,54 +259,17 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { R"([[1;[%true];[%false]];[1;[%true];[%true]];[2;[%true];[%false]];[2;[%true];[%true]]])", Scan); } - Y_UNIT_TEST(TestJoinByBool, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString t1 = "/Root/Table1"; - const TString t2 = "/Root/Table2"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); - TTestHelper::TColumnTable col1, col2; - TVector<TTestHelper::TColumnSchema> s1, s2; - if (Table == ETableKind::COLUMNSHARD) { - s1 = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("b").SetType(NScheme::NTypeIds::Bool), - }; - - col1.SetName(t1).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s1); - helper.CreateTable(col1); - s2 = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("table1_id").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("b").SetType(NScheme::NTypeIds::Bool), - }; - - col2.SetName(t2).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s2); - helper.CreateTable(col2); - } else { - CreateDataShardTable(helper, t1); - CreateDataShardTable(helper, t2); - } - - LoadData(helper, Table, Load, t1, { { 2, 3, true }, { 4, 1, true } }, &col1, &s1); - LoadData(helper, Table, Load, t2, { { 2, 2, false }, { 4, 4, false }, { 1, 1, true }, { 3, 3, true } }, &col2, &s2); - CheckOrExec(helper, - "SELECT t1.id, t2.id, t1.b FROM `/Root/Table1` as t1 join `/Root/Table2` as t2 on t1.b = t2.b order by t1.id, t2.id, t1.b", - R"([[2;1;[%true]];[2;3;[%true]];[4;1;[%true]];[4;3;[%true]]])", Scan); - } + Y_UNIT_TEST_JOIN_SCENARIO(TestJoinByBool, JoinByBoolScenario); Y_UNIT_TEST(TestCSVBoolFormats, EQueryMode, ETableKind) { const auto Scan = Arg<0>(); const auto Table = Arg<1>(); const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); TTestHelper::TColumnTable col; TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); + Base::PrepareBase(helper, Table, tableName, &col, &schema); struct TCSVFormat { TString TrueValue; @@ -668,15 +308,15 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { TVector<TRow> rows = { { 1, 100, true }, { 2, 200, false } }; if (Table == ETableKind::DATASHARD) { - PrepareBase(helperDisabled, Table, tableName, &col, &schema); - LoadData(helperDisabled, Table, ELoadKind::ARROW, tableName, rows, &col, &schema); + Base::PrepareBase(helperDisabled, Table, tableName, &col, &schema); + Base::LoadData(helperDisabled, Table, ELoadKind::ARROW, tableName, rows, &col, &schema); CheckOrExec( helperDisabled, "SELECT id, int, b FROM `" + tableName + "` ORDER BY id", "[[1;[100];[%true]];[2;[200];[%false]]]", Scan); } else { bool tableCreated = TryPrepareBase(helperDisabled, Table, tableName, &col, &schema); if (tableCreated) { try { - LoadData(helperDisabled, Table, ELoadKind::ARROW, tableName, rows, &col, &schema); + Base::LoadData(helperDisabled, Table, ELoadKind::ARROW, tableName, rows, &col, &schema); UNIT_ASSERT_C(false, "Expected error for ColumnShard with disabled feature flag"); } catch (const std::exception& e) { TString errorMsg = e.what(); @@ -686,11 +326,11 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { } } - TTestHelper helperEnabled(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helperEnabled(TBoolTraits::CreateSettings()); TTestHelper::TColumnTable col2; TVector<TTestHelper::TColumnSchema> schema2; - PrepareBase(helperEnabled, Table, tableName, &col2, &schema2); - LoadData(helperEnabled, Table, ELoadKind::ARROW, tableName, rows, &col2, &schema2); + Base::PrepareBase(helperEnabled, Table, tableName, &col2, &schema2); + Base::LoadData(helperEnabled, Table, ELoadKind::ARROW, tableName, rows, &col2, &schema2); CheckOrExec(helperEnabled, "SELECT id, int, b FROM `" + tableName + "` ORDER BY id", "[[1;[100];[%true]];[2;[200];[%false]]]", Scan); } @@ -704,7 +344,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { const TVector<TRow>& rows, const TString& selectExpr, const TString& expectedScan) -> decltype(auto) { - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); TTestHelper::TColumnTable col; col.SetName(name).SetPrimaryKey(pkColumns).SetSharding(pkColumns).SetSchema(schema); helper.CreateTable(col); @@ -713,7 +353,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { if (Load == ELoadKind::ARROW) { using namespace NKikimr::NKqp::NTestArrow; std::vector<bool> bs; bs.reserve(rows.size()); - for (auto&& r : rows) bs.push_back(r.B.value()); + for (auto&& r : rows) bs.push_back(r.TypedVal.value()); auto bArr = MakeBoolArrayAsUInt8(bs); auto batch = MakeBatch({ arrow::field("b", arrow::uint8(), /*nullable*/ false) }, { bArr }); helper.BulkUpsert(col, batch); @@ -721,7 +361,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { TValueBuilder builder; builder.BeginList(); for (auto&& r : rows) { - builder.AddListItem().BeginStruct().AddMember("b").Bool(r.B.value()).EndStruct(); + builder.AddListItem().BeginStruct().AddMember("b").Bool(r.TypedVal.value()).EndStruct(); } builder.EndList(); @@ -730,7 +370,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { } else { TStringBuilder csv; for (auto&& r : rows) { - csv << (r.B.value() ? "true" : "false") << "\n"; + csv << (r.TypedVal.value() ? "true" : "false") << "\n"; } auto res = helper.GetKikimr().GetTableClient().BulkUpsert(name, EDataFormat::CSV, csv).GetValueSync(); @@ -744,7 +384,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { std::vector<int32_t> ids; ids.reserve(rows.size()); std::vector<uint8_t> bools; bools.reserve(rows.size()); std::vector<int64_t> ints; ints.reserve(rows.size()); - for (auto&& r : rows) { ids.push_back(r.Id); bools.push_back(r.B.value() ? 1u : 0u); ints.push_back(r.IntVal); } + for (auto&& r : rows) { ids.push_back(r.Id); bools.push_back(r.TypedVal.value() ? 1u : 0u); ints.push_back(r.IntVal); } auto idArr = MakeInt32Array(ids); auto bArr = MakeUInt8Array(bools); auto iArr = MakeInt64Array(ints); @@ -756,7 +396,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { for (auto&& r : rows) { builder.AddListItem().BeginStruct() .AddMember("id").Int32(r.Id) - .AddMember("b").Bool(r.B.value()) + .AddMember("b").Bool(r.TypedVal.value()) .AddMember("int").Int64(r.IntVal) .EndStruct(); } @@ -767,7 +407,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { } else { TStringBuilder csv; for (auto&& r : rows) { - csv << r.Id << "," << (r.B.value() ? "true" : "false") << "," << r.IntVal << "\n"; + csv << r.Id << "," << (r.TypedVal.value() ? "true" : "false") << "," << r.IntVal << "\n"; } auto res = helper.GetKikimr().GetTableClient().BulkUpsert(name, EDataFormat::CSV, csv).GetValueSync(); @@ -806,14 +446,14 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - auto settings = CreateKikimrSettingsWithBoolSupport(); + auto settings = TBoolTraits::CreateSettings(); settings.AppConfig.MutableTableServiceConfig()->SetEnableHtapTx(true); TTestHelper helper(settings); const TString ds = "/Root/RowSrc"; const TString cs = "/Root/ColSrc"; - CreateDataShardTable(helper, ds); + Base::CreateDataShardTable(helper, ds); TVector<TTestHelper::TColumnSchema> schema = { TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), @@ -903,7 +543,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); const TString cs = "/Root/BoolOps"; @@ -961,7 +601,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); const TString cs = "/Root/BoolOrderLimit"; @@ -1059,7 +699,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { Y_UNIT_TEST(TestBoolCompare, EQueryMode, ELoadKind) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); const TString cs = "/Root/BoolWhereCmp"; @@ -1139,7 +779,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { Y_UNIT_TEST(TestBoolFilterWithColumns, EQueryMode, ELoadKind) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); const TString cs = "/Root/BoolFilterCols"; @@ -1222,7 +862,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); const TString cs = "/Root/BoolWriteCmp"; @@ -1277,7 +917,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); const TString cs = "/Root/BoolNot"; @@ -1323,7 +963,7 @@ Y_UNIT_TEST_SUITE(KqpBoolColumnShard) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - TTestHelper helper(CreateKikimrSettingsWithBoolSupport()); + TTestHelper helper(TBoolTraits::CreateSettings()); const TString cs = "/Root/BoolGroup"; diff --git a/ydb/core/kqp/ut/olap/types/column_type_scenarios.h b/ydb/core/kqp/ut/olap/types/column_type_scenarios.h new file mode 100644 index 00000000000..1ce51e8c6b7 --- /dev/null +++ b/ydb/core/kqp/ut/olap/types/column_type_scenarios.h @@ -0,0 +1,165 @@ +#pragma once + +#include "column_type_test_base.h" + +namespace NKikimr::NKqp { + +struct TQueryCheck { + TString Query; + TString Expected; +}; + +template <typename TValue> +struct TScenarioLoad { + TVector<TTypedRow<TValue>> Data; + TVector<TQueryCheck> ChecksAfter; +}; + +template <typename TValue> +struct TScenario { + TString TableName; + TVector<TTypedRow<TValue>> Data; + TVector<TQueryCheck> Checks; + TVector<TScenarioLoad<TValue>> ExtraLoads{}; +}; + +template <typename TValue> +struct TJoinScenario { + TString Table1Name; + TString Table2Name; + TVector<TTypedRow<TValue>> Table1Data; + TVector<TTypedRow<TValue>> Table2Data; + TVector<TQueryCheck> Checks; +}; + +struct TPkLookupScenario { + TString TableName; + TVector<TQueryCheck> Checks; +}; + +struct TCsvScenario { + TString TableName; + TString CsvData; + TVector<TQueryCheck> Checks; +}; + +template <typename TTraits> +void RunScenario(const TScenario<typename TTraits::TValue>& scenario, + EQueryMode scan, ETableKind table, ELoadKind load) { + using Base = TColumnTypeTestBase<TTraits>; + + TTestHelper helper(TTraits::CreateSettings()); + TTestHelper::TColumnTable col; + TVector<TTestHelper::TColumnSchema> schema; + Base::PrepareBase(helper, table, scenario.TableName, &col, &schema); + Base::LoadData(helper, table, load, scenario.TableName, scenario.Data, &col, &schema); + + for (auto&& check : scenario.Checks) { + CheckOrExec(helper, check.Query, check.Expected, scan); + } + + for (auto&& extra : scenario.ExtraLoads) { + Base::LoadData(helper, table, load, scenario.TableName, extra.Data, &col, &schema); + for (auto&& check : extra.ChecksAfter) { + CheckOrExec(helper, check.Query, check.Expected, scan); + } + } +} + +template <typename TTraits> +void RunJoinScenario(const TJoinScenario<typename TTraits::TValue>& scenario, + EQueryMode scan, ETableKind table, ELoadKind load) { + using Base = TColumnTypeTestBase<TTraits>; + + TTestHelper helper(TTraits::CreateSettings()); + TTestHelper::TColumnTable col1; + TTestHelper::TColumnTable col2; + TVector<TTestHelper::TColumnSchema> s1; + TVector<TTestHelper::TColumnSchema> s2; + + if (table == ETableKind::COLUMNSHARD) { + TVector<TTestHelper::TColumnSchema> schema = { + TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), + TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), + TTestHelper::TColumnSchema().SetName(TTraits::ColumnName).SetType(TTraits::GetTypeId()), + }; + + s1 = schema; + col1.SetName(scenario.Table1Name).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema); + helper.CreateTable(col1); + s2 = schema; + col2.SetName(scenario.Table2Name).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema); + helper.CreateTable(col2); + } else { + Base::CreateDataShardTable(helper, scenario.Table1Name); + Base::CreateDataShardTable(helper, scenario.Table2Name); + } + + Base::LoadData(helper, table, load, scenario.Table1Name, scenario.Table1Data, &col1, &s1); + Base::LoadData(helper, table, load, scenario.Table2Name, scenario.Table2Data, &col2, &s2); + + for (auto&& check : scenario.Checks) { + CheckOrExec(helper, check.Query, check.Expected, scan); + } +} + +template <typename TTraits> +void RunPkLookupScenario(const TPkLookupScenario& scenario, EQueryMode scan, ELoadKind load) { + TTestHelper helper(TTraits::CreateSettings()); + TVector<TTestHelper::TColumnSchema> schema = { + TTestHelper::TColumnSchema().SetName(TTraits::ColumnName).SetType(TTraits::GetTypeId()).SetNullable(false), + TTestHelper::TColumnSchema().SetName("val").SetType(NScheme::NTypeIds::Int64), + }; + + TTestHelper::TColumnTable testTable; + testTable.SetName(scenario.TableName) + .SetPrimaryKey({ TTraits::ColumnName }) + .SetSharding({ TTraits::ColumnName }) + .SetSchema(schema); + helper.CreateTable(testTable); + TTraits::LoadPkTable(helper, load, scenario.TableName, testTable); + + for (auto&& check : scenario.Checks) { + CheckOrExec(helper, check.Query, check.Expected, scan); + } +} + +template <typename TTraits> +void RunCsvScenario(const TCsvScenario& scenario, EQueryMode scan, ETableKind table) { + using Base = TColumnTypeTestBase<TTraits>; + + TTestHelper helper(TTraits::CreateSettings()); + TTestHelper::TColumnTable col; + TVector<TTestHelper::TColumnSchema> schema; + Base::PrepareBase(helper, table, scenario.TableName, &col, &schema); + + auto result = helper.GetKikimr().GetTableClient().BulkUpsert( + scenario.TableName, EDataFormat::CSV, scenario.CsvData).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + for (auto&& check : scenario.Checks) { + CheckOrExec(helper, check.Query, check.Expected, scan); + } +} + +#define Y_UNIT_TEST_SCENARIO(Name, ScenarioFn) \ + Y_UNIT_TEST(Name, EQueryMode, ETableKind, ELoadKind) { \ + RunScenario<TTraits>(ScenarioFn(), Arg<0>(), Arg<1>(), Arg<2>()); \ + } + +#define Y_UNIT_TEST_JOIN_SCENARIO(Name, ScenarioFn) \ + Y_UNIT_TEST(Name, EQueryMode, ETableKind, ELoadKind) { \ + RunJoinScenario<TTraits>(ScenarioFn(), Arg<0>(), Arg<1>(), Arg<2>()); \ + } + +#define Y_UNIT_TEST_PK_SCENARIO(Name, ScenarioFn) \ + Y_UNIT_TEST(Name, EQueryMode, ELoadKind) { \ + RunPkLookupScenario<TTraits>(ScenarioFn(), Arg<0>(), Arg<1>()); \ + } + +#define Y_UNIT_TEST_CSV_SCENARIO(Name, ScenarioFn) \ + Y_UNIT_TEST(Name, EQueryMode, ETableKind) { \ + RunCsvScenario<TTraits>(ScenarioFn(), Arg<0>(), Arg<1>()); \ + } + +} // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/olap/types/column_type_test_base.h b/ydb/core/kqp/ut/olap/types/column_type_test_base.h new file mode 100644 index 00000000000..b20411caabb --- /dev/null +++ b/ydb/core/kqp/ut/olap/types/column_type_test_base.h @@ -0,0 +1,299 @@ +#pragma once + +#include "column_type_test_enums.h" + +#include <ydb/core/formats/arrow/arrow_helpers.h> +#include <ydb/core/kqp/ut/common/arrow_builders.h> +#include <ydb/core/kqp/ut/common/columnshard.h> +#include <ydb/core/kqp/ut/common/kqp_ut_common.h> +#include <ydb/core/testlib/common_helper.h> +#include <ydb/core/testlib/cs_helper.h> +#include <ydb/core/tx/columnshard/hooks/testing/controller.h> +#include <ydb/core/tx/tx_proxy/proxy.h> + +#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_replication.h> +#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h> +#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h> +#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/client.h> + +#include <library/cpp/threading/local_executor/local_executor.h> +#include <util/generic/serialized_enum.h> +#include <util/string/printf.h> + +namespace NKikimr::NKqp { + +using namespace NYdb; +using namespace NYdb::NTable; + +template <typename TValue> +struct TTypedRow { + i32 Id; + i64 IntVal; + std::optional<TValue> TypedVal; +}; + +inline void CheckOrExec(TTestHelper& helper, const TString& query, const TString& expected, EQueryMode scanMode) { + if (scanMode == EQueryMode::SCAN_QUERY) { + helper.ReadData(query, expected); + } else { + helper.ReadDataExecQuery(query, expected); + } +} + +template <typename TConfigure> +TKikimrSettings CreateColumnshardSettings(TConfigure&& configure) { + NKikimrConfig::TFeatureFlags featureFlags; + configure(featureFlags); + return TKikimrSettings().SetWithSampleTables(false).SetFeatureFlags(featureFlags); +} + +// TTraits contract: +// using TValue = ...; +// static constexpr const char* ColumnName; +// static constexpr const char* SqlTypeName; +// static TKikimrSettings CreateSettings(); +// static NScheme::TTypeId GetTypeId(); +// static void AppendYdbValue(TValueBuilder& builder, const std::optional<TValue>& val); +// static void AppendCsvValue(TStringBuilder& builder, const std::optional<TValue>& val); +// static std::shared_ptr<arrow::Array> MakeArrowArray(const TVector<TTypedRow<TValue>>& rows); +// static std::shared_ptr<arrow::DataType> ArrowType(); +template <typename TTraits> +struct TColumnTypeTestBase { + using TValue = typename TTraits::TValue; + using TRow = TTypedRow<TValue>; + + static TString GetSqlTypeName() { + if constexpr (requires { { TTraits::BuildSqlTypeName() } -> std::convertible_to<TString>; }) { + return TTraits::BuildSqlTypeName(); + } else { + return TTraits::SqlTypeName; + } + } + + static void CreateDataShardTable(TTestHelper& helper, const TString& name) { + auto& session = helper.GetSession(); + auto res = session + .ExecuteSchemeQuery(TStringBuilder() << R"( + CREATE TABLE `)" << name << R"(` ( + id Int32 NOT NULL, + int Int64, + )" << TTraits::ColumnName << " " << GetSqlTypeName() << R"(, + PRIMARY KEY (id) + ); + )") + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), NYdb::EStatus::SUCCESS); + } + + static void BulkUpsertRowTableYdbValue(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { + TValueBuilder builder; + builder.BeginList(); + for (auto&& r : rows) { + builder.AddListItem().BeginStruct() + .AddMember("id").Int32(r.Id) + .AddMember("int").Int64(r.IntVal) + .AddMember(TTraits::ColumnName); + TTraits::AppendYdbValue(builder, r.TypedVal); + builder.EndStruct(); + } + + builder.EndList(); + auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, builder.Build()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + static void BulkUpsertRowTableCSV(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { + BulkUpsertRowTableCSVWithColumnName(helper, name, rows, "int"); + } + + static void BulkUpsertRowTableCSVWithColumnName( + TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& columnName) { + Y_UNUSED(columnName); + TStringBuilder builder; + for (auto&& r : rows) { + builder << r.Id << "," << r.IntVal << ","; + TTraits::AppendCsvValue(builder, r.TypedVal); + builder << '\n'; + } + + auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, EDataFormat::CSV, builder).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + static std::shared_ptr<arrow::RecordBatch> MakeArrowBatch(const TVector<TRow>& rows) { + using namespace NKikimr::NKqp::NTestArrow; + std::vector<int32_t> ids; + std::vector<int64_t> vals; + ids.reserve(rows.size()); + vals.reserve(rows.size()); + for (auto&& r : rows) { + ids.push_back(r.Id); + vals.push_back(r.IntVal); + } + + auto idArr = MakeInt32Array(ids); + auto intArr = MakeInt64Array(vals); + auto typedArr = TTraits::MakeArrowArray(rows); + return MakeBatch( + { arrow::field("id", arrow::int32(), false), + arrow::field("int", arrow::int64()), + arrow::field(TTraits::ColumnName, TTraits::ArrowType()) }, + { idArr, intArr, typedArr }); + } + + static void BulkUpsertRowTableArrow(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { + auto batch = MakeArrowBatch(rows); + TString strBatch = NArrow::SerializeBatchNoCompression(batch); + TString strSchema = NArrow::SerializeSchema(*batch->schema()); + auto result = helper.GetKikimr().GetTableClient().BulkUpsert( + name, NYdb::NTable::EDataFormat::ApacheArrow, strBatch, strSchema).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + static void LoadData(TTestHelper& helper, ETableKind table, ELoadKind load, + const TString& name, const TVector<TRow>& rows, + TTestHelper::TColumnTable* col = nullptr, + const TVector<TTestHelper::TColumnSchema>* schema = nullptr) { + switch (table) { + case ETableKind::COLUMNSHARD: { + Y_ABORT_UNLESS(col && schema); + TString columnName = "int"; + if (schema->size() >= 2) { + columnName = (*schema)[1].GetName(); + } + + if (load == ELoadKind::ARROW) { + auto batch = MakeArrowBatchWithColumnName(rows, columnName); + helper.BulkUpsert(*col, batch); + } else if (load == ELoadKind::YDB_VALUE) { + BulkUpsertRowTableYdbValueWithColumnName(helper, name, rows, columnName); + } else { + BulkUpsertRowTableCSVWithColumnName(helper, name, rows, columnName); + } + + break; + } + case ETableKind::DATASHARD: { + if (load == ELoadKind::ARROW) { + BulkUpsertRowTableArrow(helper, name, rows); + } else if (load == ELoadKind::YDB_VALUE) { + BulkUpsertRowTableYdbValue(helper, name, rows); + } else { + BulkUpsertRowTableCSV(helper, name, rows); + } + + break; + } + } + } + + static void PrepareBase(TTestHelper& helper, ETableKind tableKind, const TString& tableName, + TTestHelper::TColumnTable* colTableOut, TVector<TTestHelper::TColumnSchema>* schemaOut) { + if (tableKind == ETableKind::COLUMNSHARD) { + TVector<TTestHelper::TColumnSchema> schema = { + TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), + TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), + TTestHelper::TColumnSchema().SetName(TTraits::ColumnName).SetType(TTraits::GetTypeId()), + }; + + *schemaOut = schema; + TTestHelper::TColumnTable col; + col.SetName(tableName).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema); + helper.CreateTable(col); + *colTableOut = col; + } else { + CreateDataShardTable(helper, tableName); + } + } + + static void CreateDataShardTableWithSecondColumn(TTestHelper& helper, const TString& name, const TString& secondName) { + auto& session = helper.GetSession(); + auto res = session + .ExecuteSchemeQuery(TStringBuilder() << R"( + CREATE TABLE `)" << name << R"(` ( + id Int32 NOT NULL, + )" << secondName << R"( Int64, + )" << TTraits::ColumnName << " " << GetSqlTypeName() << R"(, + PRIMARY KEY (id) + ); + )") + .ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), NYdb::EStatus::SUCCESS); + } + + static void BulkUpsertRowTableYdbValueWithColumnName( + TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& columnName) { + TValueBuilder builder; + builder.BeginList(); + for (auto&& r : rows) { + builder.AddListItem().BeginStruct() + .AddMember("id").Int32(r.Id) + .AddMember(columnName).Int64(r.IntVal) + .AddMember(TTraits::ColumnName); + TTraits::AppendYdbValue(builder, r.TypedVal); + builder.EndStruct(); + } + + builder.EndList(); + auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, builder.Build()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + static void BulkUpsertRowTableYdbValueWithSecondColumn( + TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& secondName) { + BulkUpsertRowTableYdbValueWithColumnName(helper, name, rows, secondName); + } + + static std::shared_ptr<arrow::RecordBatch> MakeArrowBatchWithColumnName(const TVector<TRow>& rows, const TString& columnName) { + using namespace NKikimr::NKqp::NTestArrow; + std::vector<int32_t> ids; + std::vector<int64_t> vals; + ids.reserve(rows.size()); + vals.reserve(rows.size()); + for (auto&& r : rows) { + ids.push_back(r.Id); + vals.push_back(r.IntVal); + } + + auto idArr = MakeInt32Array(ids); + auto intArr = MakeInt64Array(vals); + auto typedArr = TTraits::MakeArrowArray(rows); + return MakeBatch( + { arrow::field("id", arrow::int32(), false), + arrow::field(std::string(columnName), arrow::int64()), + arrow::field(TTraits::ColumnName, TTraits::ArrowType()) }, + { idArr, intArr, typedArr }); + } + + static std::shared_ptr<arrow::RecordBatch> MakeArrowBatchWithSecondColumn(const TVector<TRow>& rows, const TString& secondName) { + return MakeArrowBatchWithColumnName(rows, secondName); + } + + static void BulkUpsertRowTableArrowWithSecondColumn( + TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& secondName) { + auto batch = MakeArrowBatchWithSecondColumn(rows, secondName); + TString strBatch = NArrow::SerializeBatchNoCompression(batch); + TString strSchema = NArrow::SerializeSchema(*batch->schema()); + auto result = helper.GetKikimr().GetTableClient().BulkUpsert( + name, NYdb::NTable::EDataFormat::ApacheArrow, strBatch, strSchema).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } +}; + +template <typename TTraits> +struct TColumnTypeTestContext { + using Traits = TTraits; + using Base = TColumnTypeTestBase<TTraits>; + using TRow = TTypedRow<typename TTraits::TValue>; + using TValue = typename TTraits::TValue; + + static TKikimrSettings Settings() { return TTraits::CreateSettings(); } +}; + +#define COLUMN_TYPE_TEST_USING(Traits) \ + using TTraits = Traits; \ + using Ctx = TColumnTypeTestContext<TTraits>; \ + using Base = Ctx::Base; \ + using TRow = Ctx::TRow; + +} // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/olap/types/bool_test_enums.h b/ydb/core/kqp/ut/olap/types/column_type_test_enums.h index 03c5a2eb7cc..03c5a2eb7cc 100644 --- a/ydb/core/kqp/ut/olap/types/bool_test_enums.h +++ b/ydb/core/kqp/ut/olap/types/column_type_test_enums.h diff --git a/ydb/core/kqp/ut/olap/types/decimal_ut.cpp b/ydb/core/kqp/ut/olap/types/decimal_ut.cpp index d9d9ae6872d..e89c3df331f 100644 --- a/ydb/core/kqp/ut/olap/types/decimal_ut.cpp +++ b/ydb/core/kqp/ut/olap/types/decimal_ut.cpp @@ -1,22 +1,5 @@ -#include <ydb/core/formats/arrow/arrow_helpers.h> -#include <ydb/core/kqp/ut/common/columnshard.h> -#include <ydb/core/kqp/ut/common/kqp_ut_common.h> -#include <ydb/core/testlib/common_helper.h> -#include <ydb/core/testlib/cs_helper.h> -#include <ydb/core/tx/columnshard/hooks/testing/controller.h> -#include <ydb/core/tx/columnshard/test_helper/test_combinator.h> -#include <ydb/core/tx/tx_proxy/proxy.h> - -#include <yql/essentials/types/binary_json/write.h> -#include <yql/essentials/types/uuid/uuid.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_replication.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/client.h> - -#include <library/cpp/threading/local_executor/local_executor.h> -#include <util/generic/serialized_enum.h> -#include <util/string/printf.h> +#include "column_type_scenarios.h" +#include "column_type_test_base.h" namespace NKikimr { namespace NKqp { @@ -24,384 +7,544 @@ namespace NKqp { using namespace NYdb; using namespace NYdb::NTable; -enum class EQueryMode { - SCAN_QUERY, - EXECUTE_QUERY -}; - Y_UNIT_TEST_SUITE(KqpDecimalColumnShard) { - class TDecimalTestCase { - public: - TDecimalTestCase(ui32 precision, ui32 scale) - : TestHelper(TKikimrSettings().SetWithSampleTables(false)) - , Precision(precision) - , Scale(scale) - {} + namespace { - TTestHelper::TUpdatesBuilder Inserter() { - return TTestHelper::TUpdatesBuilder(TestTable.GetArrowSchema(Schema)); - } + template <ui32 Precision, ui32 Scale> + struct TDecimalTraits { + using TValue = TString; + static constexpr const char* ColumnName = "dec"; + static constexpr ui32 DecimalPrecision = Precision; + static constexpr ui32 DecimalScale = Scale; - void Upsert(TTestHelper::TUpdatesBuilder& inserter) { - TestHelper.BulkUpsert(TestTable, inserter); + static TKikimrSettings CreateSettings() { + return TKikimrSettings().SetWithSampleTables(false); } - void CheckQuery(const TString& query, const TString& expected, EQueryMode mode = EQueryMode::SCAN_QUERY) const { - switch (mode) { - case EQueryMode::SCAN_QUERY: - TestHelper.ReadData(query, expected); - break; - case EQueryMode::EXECUTE_QUERY: { - TestHelper.ExecuteQuery(query); - break; - } - } - } + static auto GetTypeId() { return NScheme::TDecimalType(Precision, Scale); } - void ExecuteDataQuery(const TString& query) const { - TestHelper.ExecuteQuery(query); + static TString BuildSqlTypeName() { + return TStringBuilder() << "Decimal(" << Precision << "," << Scale << ")"; } - void PrepareTable1() { - Schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("dec").SetType(NScheme::TDecimalType(Precision, Scale)), - }; - TestTable.SetName("/Root/Table1").SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(Schema); - TestHelper.CreateTable(TestTable); + static TString DecCast(const TString& val) { + return TStringBuilder() << "CAST(\"" << val << "\" AS Decimal(" << Precision << "," << Scale << "))"; + } - { - TTestHelper::TUpdatesBuilder inserter = Inserter(); - inserter.AddRow().Add(1).Add(4).Add(TDecimalValue("3.14", Precision, Scale)); - inserter.AddRow().Add(2).Add(3).Add(TDecimalValue("8.16", Precision, Scale)); - Upsert(inserter); + static void AppendYdbValue(TValueBuilder& builder, const std::optional<TString>& val) { + if (val.has_value()) { + builder.BeginOptional().Decimal(TDecimalValue(*val, Precision, Scale)).EndOptional(); + } else { + builder.EmptyOptional(NYdb::TTypeBuilder().Decimal(TDecimalType(Precision, Scale)).Build()); } - { - TTestHelper::TUpdatesBuilder inserter = Inserter(); - inserter.AddRow().Add(4).Add(1).Add(TDecimalValue("12.46", Precision, Scale)); - inserter.AddRow().Add(3).Add(2).Add(TDecimalValue("8.492", Precision, Scale)); + } - Upsert(inserter); + static void AppendCsvValue(TStringBuilder& builder, const std::optional<TString>& val) { + if (val.has_value()) { + builder << *val; } } - void PrepareTable2() { - Schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("table1_id").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("dec").SetType(NScheme::TDecimalType(Precision, Scale)), - }; - TestTable.SetName("/Root/Table2").SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(Schema); - TestHelper.CreateTable(TestTable); - - { - TTestHelper::TUpdatesBuilder inserter = Inserter(); - inserter.AddRow().Add(1).Add(1).Add(TDecimalValue("12.46", Precision, Scale)); - inserter.AddRow().Add(2).Add(1).Add(TDecimalValue("8.16", Precision, Scale)); - inserter.AddRow().Add(3).Add(2).Add(TDecimalValue("12.46", Precision, Scale)); - inserter.AddRow().Add(4).Add(2).Add(TDecimalValue("8.16", Precision, Scale)); - Upsert(inserter); + static void AppendDecimalBytes(arrow::FixedSizeBinaryBuilder& builder, const TString& val) { + TDecimalValue decimal(val, Precision, Scale); + char bytes[NScheme::FSB_SIZE] = { 0 }; + for (i32 i = 0; i < 8; ++i) { + bytes[i] = (decimal.Low_ >> (i << 3)) & 0xFF; + bytes[i + 8] = (decimal.Hi_ >> (i << 3)) & 0xFF; } + Y_ABORT_UNLESS(builder.Append(bytes).ok()); } - private: - TTestHelper TestHelper; + static std::shared_ptr<arrow::Array> MakeArrowArray(const TVector<TTypedRow<TString>>& rows) { + auto type = arrow::fixed_size_binary(NScheme::FSB_SIZE); + arrow::FixedSizeBinaryBuilder builder(type); + for (auto&& r : rows) { + if (r.TypedVal.has_value()) { + AppendDecimalBytes(builder, *r.TypedVal); + } else { + Y_ABORT_UNLESS(builder.AppendNull().ok()); + } + } + std::shared_ptr<arrow::Array> arr; + Y_ABORT_UNLESS(builder.Finish(&arr).ok()); + return arr; + } - TVector<TTestHelper::TColumnSchema> Schema; - TTestHelper::TColumnTable TestTable; + static std::shared_ptr<arrow::DataType> ArrowType() { + return arrow::fixed_size_binary(NScheme::FSB_SIZE); + } - YDB_READONLY_DEF(ui32, Precision); - YDB_READONLY_DEF(ui32, Scale); + static void LoadPkTable(TTestHelper& helper, ELoadKind load, const TString& tableName, TTestHelper::TColumnTable& table); }; - Y_UNIT_TEST_DUO(TestSimpleQueries, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - TDecimalTestCase tester35(35, 10); - tester35.PrepareTable1(); - TDecimalTestCase tester12(12, 2); - tester12.PrepareTable1(); - - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; + using Decimal229 = TDecimalTraits<22, 9>; + COLUMN_TYPE_TEST_USING(Decimal229); - auto check = [mode](const TDecimalTestCase& tester) { - tester.CheckQuery("SELECT * FROM `/Root/Table1` WHERE id=1", "[[[\"3.14\"];1;[4]]]", mode); - - TString expected; - if (tester.GetPrecision() == 12 && tester.GetScale() == 2) { - expected = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.49\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } else { - expected = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } + template <> + void TDecimalTraits<22, 9>::LoadPkTable(TTestHelper& helper, ELoadKind load, const TString& tableName, TTestHelper::TColumnTable& table) { + if (load == ELoadKind::ARROW) { + auto type = arrow::fixed_size_binary(NScheme::FSB_SIZE); + arrow::FixedSizeBinaryBuilder decB(type); + Decimal229::AppendDecimalBytes(decB, "3.14"); + Decimal229::AppendDecimalBytes(decB, "8.16"); + Decimal229::AppendDecimalBytes(decB, "12.46"); + using namespace NKikimr::NKqp::NTestArrow; + auto valArr = MakeInt64Array({ (int64_t)1, (int64_t)2, (int64_t)3 }); + std::shared_ptr<arrow::Array> decArr; + Y_ABORT_UNLESS(decB.Finish(&decArr).ok()); + auto batch = MakeBatch( + { arrow::field("dec", arrow::fixed_size_binary(NScheme::FSB_SIZE), false), + arrow::field("val", arrow::int64()) }, + { decArr, valArr }); + helper.BulkUpsert(table, batch); + } else if (load == ELoadKind::YDB_VALUE) { + TValueBuilder builder; + builder.BeginList(); + builder.AddListItem().BeginStruct().AddMember("dec").Decimal(TDecimalValue("3.14", 22, 9)).AddMember("val").Int64(1).EndStruct(); + builder.AddListItem().BeginStruct().AddMember("dec").Decimal(TDecimalValue("8.16", 22, 9)).AddMember("val").Int64(2).EndStruct(); + builder.AddListItem().BeginStruct().AddMember("dec").Decimal(TDecimalValue("12.46", 22, 9)).AddMember("val").Int64(3).EndStruct(); + builder.EndList(); + auto res = helper.GetKikimr().GetTableClient().BulkUpsert(tableName, builder.Build()).GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); + } else { + TStringBuilder csv; + csv << "3.14,1\n8.16,2\n12.46,3\n"; + auto res = helper.GetKikimr().GetTableClient().BulkUpsert(tableName, EDataFormat::CSV, csv).GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); + } + } - tester.CheckQuery("SELECT * FROM `/Root/Table1` order by id", expected, mode); + TScenario<TString> FilterEqualScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, "3.14" }, + { 2, 20, "8.16" }, + { 3, 30, "8.492" }, + { 4, 40, "12.46" }, + }, + { + { "SELECT id FROM `" + tableName + "` WHERE dec = " + Decimal229::DecCast("3.14"), "[[1]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dec != " + Decimal229::DecCast("3.14") + " ORDER BY id", "[[2];[3];[4]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dec = " + Decimal229::DecCast("12.46"), "[[4]]" }, + }, }; - - check(tester22); - check(tester35); - check(tester12); } - Y_UNIT_TEST_DUO(TestFilterEqual, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - TDecimalTestCase tester35(35, 10); - tester35.PrepareTable1(); - TDecimalTestCase tester12(12, 2); - tester12.PrepareTable1(); - - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; - - auto check = [mode](const TDecimalTestCase& tester, ui32 precision, ui32 scale) { - tester.CheckQuery(TString::Join("SELECT * FROM `/Root/Table1` WHERE dec == cast(\"3.14\" as decimal(", ToString(precision), ",", ToString(scale), "))"), "[[[\"3.14\"];1;[4]]]", mode); - - TString expected; - if (precision == 12 && scale == 2) { - expected = "[[[\"8.16\"];2;[3]];[[\"8.49\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } else { - expected = "[[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } - - tester.CheckQuery(TString::Join("SELECT * FROM `/Root/Table1` WHERE dec != cast(\"3.14\" as decimal(", ToString(precision), ",", ToString(scale), ")) order by id"), expected, mode); + TScenario<TString> FilterNullsScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, "3.14" }, + { 2, 20, std::nullopt }, + { 3, 30, "8.16" }, + { 4, 40, std::nullopt }, + { 5, 50, "12.46" }, + }, + { + { "SELECT id FROM `" + tableName + "` WHERE dec IS NULL ORDER BY id", "[[2];[4]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dec IS NOT NULL ORDER BY id", "[[1];[3];[5]]" }, + }, }; - - check(tester22, 22, 9); - check(tester35, 35, 10); - check(tester12, 12, 2); } - Y_UNIT_TEST_DUO(TestFilterNulls, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - TDecimalTestCase tester35(35, 10); - tester35.PrepareTable1(); - TDecimalTestCase tester12(12, 2); - tester12.PrepareTable1(); - - auto insert = [](TDecimalTestCase& tester) { - TTestHelper::TUpdatesBuilder inserter = tester.Inserter(); - inserter.AddRow().Add(5).Add(5).AddNull(); - inserter.AddRow().Add(6).Add(6).AddNull(); - tester.Upsert(inserter); + TScenario<TString> FilterCompareScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, "3.14" }, + { 2, 20, "8.16" }, + { 3, 30, "8.492" }, + { 4, 40, "12.46" }, + }, + { + { "SELECT id FROM `" + tableName + "` WHERE dec < " + Decimal229::DecCast("12.46") + " ORDER BY id", + "[[1];[2];[3]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dec > " + Decimal229::DecCast("8.16") + " ORDER BY id", + "[[3];[4]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dec <= " + Decimal229::DecCast("12.46") + " ORDER BY id", + "[[1];[2];[3];[4]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dec >= " + Decimal229::DecCast("8.492") + " ORDER BY id", + "[[3];[4]]" }, + }, }; + } - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; - - auto check = [mode](const TDecimalTestCase& tester) { - tester.CheckQuery("SELECT * FROM `/Root/Table1` WHERE dec is NULL order by id", "[[#;5;[5]];[#;6;[6]]]", mode); - - TString expected; - if (tester.GetPrecision() == 12 && tester.GetScale() == 2) { - expected = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.49\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } else { - expected = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } - - tester.CheckQuery("SELECT * FROM `/Root/Table1` WHERE dec is not NULL order by id", expected, mode); + TScenario<TString> OrderByScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, "12.46" }, + { 2, 20, "3.14" }, + { 3, 30, "8.492" }, + { 4, 40, "8.16" }, + }, + { + { "SELECT dec, id FROM `" + tableName + "` ORDER BY dec", + "[[[\"3.14\"];2];[[\"8.16\"];4];[[\"8.492\"];3];[[\"12.46\"];1]]" }, + { "SELECT dec, id FROM `" + tableName + "` ORDER BY dec DESC", + "[[[\"12.46\"];1];[[\"8.492\"];3];[[\"8.16\"];4];[[\"3.14\"];2]]" }, + }, }; - - insert(tester22); - insert(tester35); - insert(tester12); - - check(tester22); - check(tester35); - check(tester12); } - Y_UNIT_TEST_DUO(TestFilterCompare, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - TDecimalTestCase tester35(35, 10); - tester35.PrepareTable1(); - TDecimalTestCase tester12(12, 2); - tester12.PrepareTable1(); + TScenario<TString> GroupByScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, "3.14" }, + { 2, 20, "8.16" }, + { 3, 30, "3.14" }, + { 4, 40, "8.16" }, + { 5, 50, "12.46" }, + }, + { + { "SELECT dec, count(*) AS cnt FROM `" + tableName + "` GROUP BY dec ORDER BY dec", + "[[[\"3.14\"];2u];[[\"8.16\"];2u];[[\"12.46\"];1u]]" }, + }, + }; + } - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; + TScenario<TString> AggregationScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, "3.14" }, + { 2, 20, "8.16" }, + { 3, 30, "8.492" }, + { 4, 40, std::nullopt }, + { 5, 50, "12.46" }, + }, + { + { "SELECT min(dec) FROM `" + tableName + "`", "[[[\"3.14\"]]]" }, + { "SELECT max(dec) FROM `" + tableName + "`", "[[[\"12.46\"]]]" }, + { "SELECT count(dec) FROM `" + tableName + "`", "[[4u]]" }, + { "SELECT count(*) FROM `" + tableName + "`", "[[5u]]" }, + { "SELECT sum(dec) FROM `" + tableName + "`", "[[[\"32.252\"]]]" }, + }, + }; + } - auto check = [mode](const TDecimalTestCase& tester, ui32 precision, ui32 scale) { - TString expected1, expected2, expected3, expected4; - - if (precision == 12 && scale == 2) { - expected1 = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.49\"];3;[2]]]"; - expected2 = "[[[\"8.49\"];3;[2]];[[\"12.46\"];4;[1]]]"; - expected3 = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.49\"];3;[2]];[[\"12.46\"];4;[1]]]"; - expected4 = "[[[\"8.49\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } else { - expected1 = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]]]"; - expected2 = "[[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]"; - expected3 = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]"; - expected4 = "[[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } - - tester.CheckQuery(TString::Join("SELECT * FROM `/Root/Table1` WHERE dec < cast(\"12.46\" as decimal(", ToString(precision), ",", ToString(scale), ")) order by id"), expected1, mode); + TJoinScenario<TString> JoinByDecimalScenario() { + const TString t1 = "/Root/Table1"; + const TString t2 = "/Root/Table2"; + return { + t1, + t2, + { + { 2, 20, "8.16" }, + { 4, 40, "12.46" }, + }, + { + { 1, 100, "12.46" }, + { 2, 200, "8.16" }, + { 3, 300, "12.46" }, + { 4, 400, "8.16" }, + }, + { + { "SELECT t1.id, t2.id, t1.dec FROM `" + t1 + "` AS t1 " + "JOIN `" + t2 + "` AS t2 ON t1.dec = t2.dec ORDER BY t1.id, t2.id", + R"([[2;2;["8.16"]];[2;4;["8.16"]];[4;1;["12.46"]];[4;3;["12.46"]]])" }, + }, + }; + } - tester.CheckQuery(TString::Join("SELECT * FROM `/Root/Table1` WHERE dec > cast(\"8.16\" as decimal(", ToString(precision), ",", ToString(scale), ")) order by id"), expected2, mode); + TScenario<TString> OrderByWithLimitScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, "12.46" }, + { 2, 20, "3.14" }, + { 3, 30, "8.16" }, + { 4, 40, "8.492" }, + { 5, 50, "100" }, + }, + { + { "SELECT dec, id FROM `" + tableName + "` ORDER BY dec LIMIT 2", + "[[[\"3.14\"];2];[[\"8.16\"];3]]" }, + { "SELECT dec, id FROM `" + tableName + "` ORDER BY dec DESC LIMIT 2", + "[[[\"100\"];5];[[\"12.46\"];1]]" }, + { "SELECT dec, id FROM `" + tableName + "` ORDER BY dec LIMIT 2 OFFSET 2", + "[[[\"8.492\"];4];[[\"12.46\"];1]]" }, + }, + }; + } - tester.CheckQuery(TString::Join("SELECT * FROM `/Root/Table1` WHERE dec <= cast(\"12.46\" as decimal(", ToString(precision), ",", ToString(scale), ")) order by id"), expected3, mode); + TScenario<TString> GroupByWithNullsScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, "3.14" }, + { 2, 20, std::nullopt }, + { 3, 30, "3.14" }, + { 4, 40, std::nullopt }, + { 5, 50, "8.16" }, + }, + { + { "SELECT dec, count(*) AS cnt FROM `" + tableName + "` GROUP BY dec ORDER BY dec", + "[[#;2u];[[\"3.14\"];2u];[[\"8.16\"];1u]]" }, + { "SELECT count(dec), count(*) FROM `" + tableName + "`", "[[3u;5u]]" }, + }, + }; + } - tester.CheckQuery(TString::Join("SELECT * FROM `/Root/Table1` WHERE dec >= cast(\"8.492\" as decimal(", ToString(precision), ",", ToString(scale), ")) order by id"), expected4, mode); + TPkLookupScenario PkLookupScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { "SELECT val FROM `" + tableName + "` WHERE dec = " + Decimal229::DecCast("8.16"), "[[[2]]]" }, + { "SELECT val FROM `" + tableName + "` WHERE dec = " + Decimal229::DecCast("3.14"), "[[[1]]]" }, + { "SELECT val FROM `" + tableName + "` WHERE dec = " + Decimal229::DecCast("12.46"), "[[[3]]]" }, + }, }; + } - check(tester22, 22, 9); - check(tester35, 35, 10); - check(tester12, 12, 2); + TCsvScenario CsvScenario() { + const TString tableName = "/Root/Table1"; + return { + tableName, + "1,10,3.14\n2,20,8.16\n3,30,8.492\n", + { + { "SELECT id FROM `" + tableName + "` ORDER BY id", "[[1];[2];[3]]" }, + { "SELECT dec, id FROM `" + tableName + "` ORDER BY id", + "[[[\"3.14\"];1];[[\"8.16\"];2];[[\"8.492\"];3]]" }, + }, + }; } - Y_UNIT_TEST_DUO(TestOrderByDecimal, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - TDecimalTestCase tester35(35, 10); - tester35.PrepareTable1(); - TDecimalTestCase tester12(12, 2); - tester12.PrepareTable1(); + template <ui32 Precision, ui32 Scale> + void RunFilterEqualForPrecision(EQueryMode scan, ETableKind table, ELoadKind load) { + using TTraits = TDecimalTraits<Precision, Scale>; + const TString tableName = TStringBuilder() << "/Root/ColumnTableTest_" << Precision << "_" << Scale; + const TString midValue = (Scale == 2) ? "8.49" : "8.492"; + TScenario<TString> scenario{ + tableName, + { + { 1, 4, "3.14" }, + { 2, 3, "8.16" }, + { 3, 2, midValue }, + { 4, 1, "12.46" }, + }, + { + { "SELECT id FROM `" + tableName + "` WHERE dec = " + TTraits::DecCast("3.14"), "[[1]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dec != " + TTraits::DecCast("3.14") + " ORDER BY id", + TStringBuilder() << "[[2];[3];[4]]" }, + }, + }; + RunScenario<TTraits>(scenario, scan, table, load); + } - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; + } // namespace - auto check = [mode](const TDecimalTestCase& tester) { - TString expected; - if (tester.GetPrecision() == 12 && tester.GetScale() == 2) { - expected = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.49\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } else { - expected = "[[[\"3.14\"];1;[4]];[[\"8.16\"];2;[3]];[[\"8.492\"];3;[2]];[[\"12.46\"];4;[1]]]"; - } + Y_UNIT_TEST(TestSimpleQueries, EQueryMode, ETableKind, ELoadKind) { + const auto Scan = Arg<0>(); + const auto Table = Arg<1>(); + const auto Load = Arg<2>(); - tester.CheckQuery("SELECT * FROM `/Root/Table1` order by dec", expected, mode); - }; + const TString tableName = "/Root/ColumnTableTest"; + TTestHelper helper(Decimal229::CreateSettings()); + TTestHelper::TColumnTable col; + TVector<TTestHelper::TColumnSchema> schema; + Base::PrepareBase(helper, Table, tableName, &col, &schema); + Base::LoadData(helper, Table, Load, tableName, { + { 1, 4, "3.14" }, + { 2, 3, "8.16" }, + { 3, 2, "8.492" }, + { 4, 1, "12.46" }, + }, &col, &schema); - check(tester22); - check(tester35); - check(tester12); + CheckOrExec(helper, "SELECT id, int FROM `" + tableName + "` WHERE id=1", "[[1;[4]]]", Scan); + CheckOrExec(helper, "SELECT id, int FROM `" + tableName + "` WHERE id=3", "[[3;[2]]]", Scan); + CheckOrExec(helper, "SELECT id, int FROM `" + tableName + "` ORDER BY id", + "[[1;[4]];[2;[3]];[3;[2]];[4;[1]]]", Scan); + CheckOrExec(helper, "SELECT dec FROM `" + tableName + "` WHERE id=1", "[[[\"3.14\"]]]", Scan); + CheckOrExec(helper, "SELECT dec FROM `" + tableName + "` WHERE id=3", "[[[\"8.492\"]]]", Scan); } - Y_UNIT_TEST_DUO(TestGroupByDecimal, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - TDecimalTestCase tester35(35, 10); - tester35.PrepareTable1(); - TDecimalTestCase tester12(12, 2); - tester12.PrepareTable1(); + Y_UNIT_TEST_SCENARIO(TestFilterEqual, FilterEqualScenario); + Y_UNIT_TEST_SCENARIO(TestFilterNulls, FilterNullsScenario); + Y_UNIT_TEST_SCENARIO(TestFilterCompare, FilterCompareScenario); + Y_UNIT_TEST_SCENARIO(TestOrderByDecimal, OrderByScenario); + Y_UNIT_TEST_SCENARIO(TestGroupByDecimal, GroupByScenario); + Y_UNIT_TEST_SCENARIO(TestAggregation, AggregationScenario); - auto insert = [](TDecimalTestCase& tester) { - TTestHelper::TUpdatesBuilder inserter = tester.Inserter(); - inserter.AddRow().Add(5).Add(12).Add(TDecimalValue("8.492", tester.GetPrecision(), tester.GetScale())); - inserter.AddRow().Add(6).Add(30).Add(TDecimalValue("12.46", tester.GetPrecision(), tester.GetScale())); - tester.Upsert(inserter); - }; + Y_UNIT_TEST(TestJoinById, EQueryMode, ETableKind, ELoadKind) { + const auto Scan = Arg<0>(); + const auto Table = Arg<1>(); + const auto Load = Arg<2>(); - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; + const TString t1 = "/Root/Table1"; + const TString t2 = "/Root/Table2"; + TTestHelper helper(Decimal229::CreateSettings()); + TTestHelper::TColumnTable col1, col2; + TVector<TTestHelper::TColumnSchema> s1, s2; + if (Table == ETableKind::COLUMNSHARD) { + s1 = { + TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), + TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), + TTestHelper::TColumnSchema().SetName("dec").SetType(NScheme::TDecimalType(22, 9)), + }; + col1.SetName(t1).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s1); + helper.CreateTable(col1); + s2 = { + TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), + TTestHelper::TColumnSchema().SetName("table1_id").SetType(NScheme::NTypeIds::Int64), + TTestHelper::TColumnSchema().SetName("dec").SetType(NScheme::TDecimalType(22, 9)), + }; + col2.SetName(t2).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s2); + helper.CreateTable(col2); + } else { + Base::CreateDataShardTable(helper, t1); + Base::CreateDataShardTableWithSecondColumn(helper, t2, "table1_id"); + } - auto check = [mode](const TDecimalTestCase& tester) { - TString expected; - if (tester.GetPrecision() == 12 && tester.GetScale() == 2) { - expected = "[[[\"3.14\"];1u];[[\"8.16\"];1u];[[\"8.49\"];2u];[[\"12.46\"];2u]]"; + Base::LoadData(helper, Table, Load, t1, { { 1, 100, "3.14" }, { 2, 200, "8.16" } }, &col1, &s1); + if (Table == ETableKind::COLUMNSHARD) { + Base::LoadData(helper, Table, Load, t2, + { { 1, 1, "8.16" }, { 2, 1, "12.46" }, { 3, 2, "8.16" }, { 4, 2, "12.46" } }, &col2, &s2); + } else { + if (Load == ELoadKind::ARROW) { + Base::BulkUpsertRowTableArrowWithSecondColumn( + helper, t2, { { 1, 1, "8.16" }, { 2, 1, "12.46" }, { 3, 2, "8.16" }, { 4, 2, "12.46" } }, "table1_id"); + } else if (Load == ELoadKind::YDB_VALUE) { + Base::BulkUpsertRowTableYdbValueWithSecondColumn( + helper, t2, { { 1, 1, "8.16" }, { 2, 1, "12.46" }, { 3, 2, "8.16" }, { 4, 2, "12.46" } }, "table1_id"); } else { - expected = "[[[\"3.14\"];1u];[[\"8.16\"];1u];[[\"8.492\"];2u];[[\"12.46\"];2u]]"; + TStringBuilder csv; + csv << "1,1,8.16\n2,1,12.46\n3,2,8.16\n4,2,12.46\n"; + auto result = helper.GetKikimr().GetTableClient().BulkUpsert(t2, EDataFormat::CSV, csv).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); } + } - tester.CheckQuery("SELECT dec, count(*) FROM `/Root/Table1` group by dec order by dec", expected, mode); - }; + CheckOrExec(helper, + R"(SELECT t1.id, t1.dec, t2.dec FROM `/Root/Table1` AS t1 JOIN `/Root/Table2` AS t2 ON t1.id = t2.table1_id ORDER BY t1.id, t2.dec)", + R"([[1;["3.14"];["8.16"]];[1;["3.14"];["12.46"]];[2;["8.16"];["8.16"]];[2;["8.16"];["12.46"]]])", Scan); + } - insert(tester22); - insert(tester35); - insert(tester12); + Y_UNIT_TEST_JOIN_SCENARIO(TestJoinByDecimal, JoinByDecimalScenario); + Y_UNIT_TEST_SCENARIO(TestOrderByWithLimit, OrderByWithLimitScenario); + Y_UNIT_TEST_SCENARIO(TestGroupByWithNulls, GroupByWithNullsScenario); + Y_UNIT_TEST_PK_SCENARIO(TestDecimalAsPrimaryKey, PkLookupScenario); + Y_UNIT_TEST_CSV_SCENARIO(TestCsv, CsvScenario); - check(tester22); - check(tester35); - check(tester12); - } + Y_UNIT_TEST(TestDmlParityAndCTAS, EQueryMode, ELoadKind) { + const auto Scan = Arg<0>(); + const auto Load = Arg<1>(); - Y_UNIT_TEST_DUO(TestAggregation, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - TDecimalTestCase tester35(35, 10); - tester35.PrepareTable1(); - TDecimalTestCase tester12(12, 2); - tester12.PrepareTable1(); + auto runnerSettings = Decimal229::CreateSettings(); + runnerSettings.AppConfig.MutableTableServiceConfig()->SetEnableHtapTx(true); + TTestHelper helper(runnerSettings); - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; + const TString ds = "/Root/RowSrc"; + const TString cs = "/Root/ColSrc"; - auto check = [mode](const TDecimalTestCase& tester) { - tester.CheckQuery("SELECT min(dec) FROM `/Root/Table1`", "[[[\"3.14\"]]]", mode); - tester.CheckQuery("SELECT max(dec) FROM `/Root/Table1`", "[[[\"12.46\"]]]", mode); - - TString expectedSum; - if (tester.GetPrecision() == 12 && tester.GetScale() == 2) { - expectedSum = "[[[\"32.25\"]]]"; - } else { - expectedSum = "[[[\"32.252\"]]]"; - } + Base::CreateDataShardTable(helper, ds); - tester.CheckQuery("SELECT sum(dec) FROM `/Root/Table1`", expectedSum, mode); + TVector<TTestHelper::TColumnSchema> schema = { + TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), + TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), + TTestHelper::TColumnSchema().SetName("dec").SetType(NScheme::TDecimalType(22, 9)), }; - check(tester22); - check(tester35); - check(tester12); - } + TTestHelper::TColumnTable col; + col.SetName(cs).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema); + helper.CreateTable(col); - Y_UNIT_TEST_DUO(TestJoinById, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - tester22.PrepareTable2(); - TDecimalTestCase tester35(35, 10); - tester35.PrepareTable1(); - tester35.PrepareTable2(); - TDecimalTestCase tester12(12, 2); - tester12.PrepareTable1(); - tester12.PrepareTable2(); + helper.ExecuteQuery( + "UPSERT INTO `" + ds + "` (id, int, dec) VALUES " + "(1, 100, CAST(\"3.14\" AS Decimal(22,9))), " + "(2, 200, CAST(\"8.16\" AS Decimal(22,9)))"); - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; + if (Load == ELoadKind::ARROW) { + auto batch = Base::MakeArrowBatch({ + { 1, 100, "3.14" }, + { 2, 200, "8.16" }, + }); + helper.BulkUpsert(col, batch); + } else if (Load == ELoadKind::YDB_VALUE) { + Base::BulkUpsertRowTableYdbValue(helper, cs, { + { 1, 100, "3.14" }, + { 2, 200, "8.16" }, + }); + } else { + Base::BulkUpsertRowTableCSV(helper, cs, { + { 1, 100, "3.14" }, + { 2, 200, "8.16" }, + }); + } - auto check = [mode](const TDecimalTestCase& tester) { - tester.CheckQuery( - "SELECT t1.id, t1.dec, t2.dec FROM `/Root/Table1` as t1 join `/Root/Table2` as t2 on t1.id = t2.table1_id order by t1.id, t1.dec, " - "t2.dec", - R"([[1;["3.14"];["8.16"]];[1;["3.14"];["12.46"]];[2;["8.16"];["8.16"]];[2;["8.16"];["12.46"]]])", mode); - }; + CheckOrExec(helper, "SELECT id, int, dec FROM `" + ds + "` ORDER BY id", + "[[1;[100];[\"3.14\"]];[2;[200];[\"8.16\"]]]", Scan); + CheckOrExec(helper, "SELECT id, int, dec FROM `" + cs + "` ORDER BY id", + "[[1;[100];[\"3.14\"]];[2;[200];[\"8.16\"]]]", Scan); + + helper.ExecuteQuery( + "UPSERT INTO `" + ds + "` (id, int, dec) VALUES " + "(3, 300, CAST(\"12.46\" AS Decimal(22,9))), " + "(1, 110, CAST(\"10.1\" AS Decimal(22,9)))"); + helper.ExecuteQuery( + "UPSERT INTO `" + cs + "` (id, int, dec) VALUES " + "(3, 300, CAST(\"12.46\" AS Decimal(22,9))), " + "(1, 110, CAST(\"10.1\" AS Decimal(22,9)))"); - check(tester22); - check(tester35); - check(tester12); + CheckOrExec(helper, "SELECT id, int, dec FROM `" + ds + "` ORDER BY id", + "[[1;[110];[\"10.1\"]];[2;[200];[\"8.16\"]];[3;[300];[\"12.46\"]]]", Scan); + CheckOrExec(helper, "SELECT id, int, dec FROM `" + cs + "` ORDER BY id", + "[[1;[110];[\"10.1\"]];[2;[200];[\"8.16\"]];[3;[300];[\"12.46\"]]]", Scan); + + helper.ExecuteQuery("DELETE FROM `" + ds + "` WHERE id = 2"); + helper.ExecuteQuery("DELETE FROM `" + cs + "` WHERE id = 2"); + CheckOrExec(helper, "SELECT id FROM `" + ds + "` ORDER BY id", "[[1];[3]]", Scan); + CheckOrExec(helper, "SELECT id FROM `" + cs + "` ORDER BY id", "[[1];[3]]", Scan); } - Y_UNIT_TEST_DUO(TestJoinByDecimal, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - tester22.PrepareTable2(); - TDecimalTestCase tester35(35, 10); - tester35.PrepareTable1(); - tester35.PrepareTable2(); - TDecimalTestCase tester12(12, 2); - tester12.PrepareTable1(); - tester12.PrepareTable2(); + Y_UNIT_TEST(TestPMInfDecimal, EQueryMode, ETableKind, ELoadKind) { + const auto Scan = Arg<0>(); + const auto Table = Arg<1>(); + const auto Load = Arg<2>(); - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; + const TString tableName = "/Root/Table1"; + TTestHelper helper(Decimal229::CreateSettings()); + TTestHelper::TColumnTable col; + TVector<TTestHelper::TColumnSchema> schema; + Base::PrepareBase(helper, Table, tableName, &col, &schema); - auto check = [mode](const TDecimalTestCase& tester) { - tester.CheckQuery( - "SELECT t1.id, t2.id, t1.dec FROM `/Root/Table1` as t1 join `/Root/Table2` as t2 on t1.dec = t2.dec order by t1.id, t2.id, t1.dec", - R"([[2;2;["8.16"]];[2;4;["8.16"]];[4;1;["12.46"]];[4;3;["12.46"]]])", mode); - }; + if (Load == ELoadKind::CSV) { + Base::LoadData(helper, Table, ELoadKind::CSV, tableName, { + { 1, 10, "3.14" }, + { 2, 20, "8.16" }, + }, &col, &schema); + Base::LoadData(helper, Table, ELoadKind::YDB_VALUE, tableName, { + { 3, 30, "inf" }, + { 4, 40, "-inf" }, + }, &col, &schema); + } else { + Base::LoadData(helper, Table, Load, tableName, { + { 1, 10, "3.14" }, + { 2, 20, "8.16" }, + { 3, 30, "inf" }, + { 4, 40, "-inf" }, + }, &col, &schema); + } - check(tester22); - check(tester35); - check(tester12); + CheckOrExec(helper, "SELECT max(dec) FROM `" + tableName + "`", "[[[\"inf\"]]]", Scan); + CheckOrExec(helper, "SELECT min(dec) FROM `" + tableName + "`", "[[[\"-inf\"]]]", Scan); } - Y_UNIT_TEST_DUO(TestPMInfDecimal, UseScanQuery) { - TDecimalTestCase tester22(22, 9); - tester22.PrepareTable1(); - auto inserter = tester22.Inserter(); - inserter.AddRow().Add(1).Add(5).Add(TDecimalValue("999999999999999999999", 22, 9)); - inserter.AddRow().Add(2).Add(6).Add(TDecimalValue("-999999999999999999999", 22, 9)); - tester22.Upsert(inserter); - - EQueryMode mode = UseScanQuery ? EQueryMode::SCAN_QUERY : EQueryMode::EXECUTE_QUERY; - - tester22.CheckQuery("SELECT max(dec) FROM `/Root/Table1`", "[[[\"inf\"]]]", mode); - tester22.CheckQuery("SELECT min(dec) FROM `/Root/Table1`", "[[[\"-inf\"]]]", mode); + Y_UNIT_TEST(TestFilterEqualMultiplePrecisions, EQueryMode, ETableKind, ELoadKind) { + RunFilterEqualForPrecision<22, 9>(Arg<0>(), Arg<1>(), Arg<2>()); + RunFilterEqualForPrecision<35, 10>(Arg<0>(), Arg<1>(), Arg<2>()); + RunFilterEqualForPrecision<12, 2>(Arg<0>(), Arg<1>(), Arg<2>()); } } diff --git a/ydb/core/kqp/ut/olap/types/dynumber_ut.cpp b/ydb/core/kqp/ut/olap/types/dynumber_ut.cpp index b9db096ed09..a64ef4a221f 100644 --- a/ydb/core/kqp/ut/olap/types/dynumber_ut.cpp +++ b/ydb/core/kqp/ut/olap/types/dynumber_ut.cpp @@ -1,25 +1,8 @@ -#include "bool_test_enums.h" - -#include <ydb/core/formats/arrow/arrow_helpers.h> -#include <ydb/core/kqp/ut/common/columnshard.h> -#include <ydb/core/kqp/ut/common/kqp_ut_common.h> -#include <ydb/core/testlib/common_helper.h> -#include <ydb/core/testlib/cs_helper.h> -#include <ydb/core/tx/columnshard/hooks/testing/controller.h> -#include <ydb/core/tx/tx_proxy/proxy.h> - -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_replication.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/client.h> +#include "column_type_scenarios.h" +#include "column_type_test_base.h" #include <yql/essentials/types/dynumber/dynumber.h> -#include <library/cpp/threading/local_executor/local_executor.h> -#include <util/generic/serialized_enum.h> -#include <util/string/printf.h> -#include <ydb/core/kqp/ut/common/arrow_builders.h> - namespace NKikimr { namespace NKqp { @@ -29,178 +12,290 @@ using namespace NYdb::NTable; Y_UNIT_TEST_SUITE(KqpDyNumberColumnShard) { namespace { - struct TRow { - i32 Id; - i64 IntVal; - std::optional<TString> Dyn; - }; + struct TDyNumberTraits { + using TValue = TString; + static constexpr const char* ColumnName = "dyn"; + static constexpr const char* SqlTypeName = "DyNumber"; - static TString MakeDyNumber(const TStringBuf& str) { - auto result = NDyNumber::ParseDyNumberString(str); - Y_ABORT_UNLESS(result.Defined(), "Failed to parse DyNumber: %s", TString(str).c_str()); - return TString(str); - } - - TKikimrSettings CreateKikimrSettingsWithDyNumberSupport() { - NKikimrConfig::TFeatureFlags featureFlags; - featureFlags.SetEnableColumnshardDyNumber(true); - return TKikimrSettings().SetWithSampleTables(false).SetFeatureFlags(featureFlags); - } - - static void AppendDyNumber(arrow::BinaryBuilder& builder, const TStringBuf& str) { - Y_ABORT_UNLESS(builder.Append(str.data(), str.size()).ok()); - } + static TKikimrSettings CreateSettings() { + return CreateColumnshardSettings([](auto& f) { f.SetEnableColumnshardDyNumber(true); }); + } - void CreateDataShardTable(TTestHelper& helper, const TString& name) { - auto& session = helper.GetSession(); - auto res = session - .ExecuteSchemeQuery(TStringBuilder() << R"( - CREATE TABLE `)" << name << R"(` ( - id Int32 NOT NULL, - int Int64, - dyn DyNumber, - PRIMARY KEY (id) - ); - )") - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), NYdb::EStatus::SUCCESS); - } + static auto GetTypeId() { return NScheme::NTypeIds::DyNumber; } - void BulkUpsertRowTableYdbValue(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - TValueBuilder builder; - builder.BeginList(); - for (auto&& r : rows) { - builder.AddListItem().BeginStruct() - .AddMember("id").Int32(r.Id) - .AddMember("int").Int64(r.IntVal) - .AddMember("dyn"); - if (r.Dyn.has_value()) { - builder.BeginOptional().DyNumber(*r.Dyn).EndOptional(); + static void AppendYdbValue(TValueBuilder& builder, const std::optional<TString>& val) { + if (val.has_value()) { + builder.BeginOptional().DyNumber(*val).EndOptional(); } else { builder.EmptyOptional(EPrimitiveType::DyNumber); } - - builder.EndStruct(); } - builder.EndList(); - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, builder.Build()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void BulkUpsertRowTableCSV(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - TStringBuilder builder; - for (auto&& r : rows) { - builder << r.Id << "," << r.IntVal << ","; - if (r.Dyn.has_value()) { - builder << *r.Dyn; + static void AppendCsvValue(TStringBuilder& builder, const std::optional<TString>& val) { + if (val.has_value()) { + builder << *val; } + } - builder << '\n'; + static std::shared_ptr<arrow::Array> MakeArrowArray(const TVector<TTypedRow<TString>>& rows) { + arrow::BinaryBuilder dynBuilder; + for (auto&& r : rows) { + if (r.TypedVal.has_value()) { + Y_ABORT_UNLESS(dynBuilder.Append(r.TypedVal->data(), r.TypedVal->size()).ok()); + } else { + Y_ABORT_UNLESS(dynBuilder.AppendNull().ok()); + } + } + std::shared_ptr<arrow::Array> arr; + Y_ABORT_UNLESS(dynBuilder.Finish(&arr).ok()); + return arr; } - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, EDataFormat::CSV, builder).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } + static std::shared_ptr<arrow::DataType> ArrowType() { return arrow::binary(); } - std::shared_ptr<arrow::RecordBatch> MakeArrowBatch(const TVector<TRow>& rows) { - using namespace NKikimr::NKqp::NTestArrow; - std::vector<int32_t> ids; - std::vector<int64_t> vals; - ids.reserve(rows.size()); - vals.reserve(rows.size()); - for (auto&& r : rows) { - ids.push_back(r.Id); - vals.push_back(r.IntVal); - } + static void LoadPkTable(TTestHelper& helper, ELoadKind load, const TString& tableName, TTestHelper::TColumnTable& table); + }; - auto idArr = MakeInt32Array(ids); - auto intArr = MakeInt64Array(vals); + COLUMN_TYPE_TEST_USING(TDyNumberTraits); - arrow::BinaryBuilder dynBuilder; - for (auto&& r : rows) { - if (r.Dyn.has_value()) { - Y_ABORT_UNLESS(dynBuilder.Append(r.Dyn->data(), r.Dyn->size()).ok()); - } else { - Y_ABORT_UNLESS(dynBuilder.AppendNull().ok()); - } + static TString MakeDyNumber(const TStringBuf& str) { + auto result = NDyNumber::ParseDyNumberString(str); + Y_ABORT_UNLESS(result.Defined(), "Failed to parse DyNumber: %s", TString(str).c_str()); + return TString(str); + } + + static void AppendDyNumber(arrow::BinaryBuilder& builder, const TStringBuf& str) { + Y_ABORT_UNLESS(builder.Append(str.data(), str.size()).ok()); + } + + void TDyNumberTraits::LoadPkTable(TTestHelper& helper, ELoadKind load, const TString& tableName, TTestHelper::TColumnTable& table) { + if (load == ELoadKind::ARROW) { + arrow::BinaryBuilder dynB; + arrow::Int64Builder valB; + AppendDyNumber(dynB, "-100"); Y_ABORT_UNLESS(valB.Append(1).ok()); + AppendDyNumber(dynB, "0"); Y_ABORT_UNLESS(valB.Append(2).ok()); + AppendDyNumber(dynB, "3.14"); Y_ABORT_UNLESS(valB.Append(3).ok()); + AppendDyNumber(dynB, "100"); Y_ABORT_UNLESS(valB.Append(4).ok()); + AppendDyNumber(dynB, "999"); Y_ABORT_UNLESS(valB.Append(5).ok()); + std::shared_ptr<arrow::Array> dynArr; + std::shared_ptr<arrow::Array> valArr; + Y_ABORT_UNLESS(dynB.Finish(&dynArr).ok()); + Y_ABORT_UNLESS(valB.Finish(&valArr).ok()); + auto aSchema = arrow::schema({ + arrow::field("dyn", arrow::binary(), false), + arrow::field("val", arrow::int64()) + }); + auto batch = arrow::RecordBatch::Make(aSchema, 5, { dynArr, valArr }); + helper.BulkUpsert(table, batch); + } else if (load == ELoadKind::YDB_VALUE) { + TValueBuilder builder; + builder.BeginList(); + builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("-100").AddMember("val").BeginOptional().Int64(1).EndOptional().EndStruct(); + builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("0").AddMember("val").BeginOptional().Int64(2).EndOptional().EndStruct(); + builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("3.14").AddMember("val").BeginOptional().Int64(3).EndOptional().EndStruct(); + builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("100").AddMember("val").BeginOptional().Int64(4).EndOptional().EndStruct(); + builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("999").AddMember("val").BeginOptional().Int64(5).EndOptional().EndStruct(); + builder.EndList(); + auto res = helper.GetKikimr().GetTableClient().BulkUpsert(tableName, builder.Build()).GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); + } else { + TStringBuilder csv; + csv << "-.1e3,1\n.0,2\n.314e1,3\n.1e3,4\n.999e3,5\n"; + auto res = helper.GetKikimr().GetTableClient().BulkUpsert(tableName, EDataFormat::CSV, csv).GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); } + } - std::shared_ptr<arrow::Array> dynArr; - Y_ABORT_UNLESS(dynBuilder.Finish(&dynArr).ok()); + TScenario<TString> FilterEqualScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, MakeDyNumber("1") }, + { 2, 20, MakeDyNumber("2") }, + { 3, 30, MakeDyNumber("3.14") }, + { 4, 40, MakeDyNumber("100") }, + }, + { + { "SELECT id FROM `" + tableName + "` WHERE dyn = CAST(\"3.14\" AS DyNumber)", "[[3]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dyn = CAST(\"1\" AS DyNumber)", "[[1]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dyn != CAST(\"3.14\" AS DyNumber) ORDER BY id", "[[1];[2];[4]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dyn != CAST(\"100\" AS DyNumber) ORDER BY id", "[[1];[2];[3]]" }, + }, + }; + } - auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), /*nullable*/ false), - arrow::field("int", arrow::int64()), - arrow::field("dyn", arrow::binary()) - }); + TScenario<TString> FilterNullsScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, MakeDyNumber("1") }, + { 2, 20, std::nullopt }, + { 3, 30, MakeDyNumber("3") }, + { 4, 40, std::nullopt }, + { 5, 50, MakeDyNumber("0") }, + }, + { + { "SELECT id FROM `" + tableName + "` WHERE dyn IS NULL ORDER BY id", "[[2];[4]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dyn IS NOT NULL ORDER BY id", "[[1];[3];[5]]" }, + }, + }; + } - return arrow::RecordBatch::Make(schema, rows.size(), { idArr, intArr, dynArr }); + TScenario<TString> FilterCompareScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, MakeDyNumber("-10") }, + { 2, 20, MakeDyNumber("0") }, + { 3, 30, MakeDyNumber("5") }, + { 4, 40, MakeDyNumber("10") }, + { 5, 50, MakeDyNumber("100") }, + }, + { + { "SELECT id FROM `" + tableName + "` WHERE dyn < CAST(\"5\" AS DyNumber) ORDER BY id", "[[1];[2]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dyn > CAST(\"5\" AS DyNumber) ORDER BY id", "[[4];[5]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dyn <= CAST(\"5\" AS DyNumber) ORDER BY id", "[[1];[2];[3]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dyn >= CAST(\"10\" AS DyNumber) ORDER BY id", "[[4];[5]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dyn > CAST(\"-10\" AS DyNumber) ORDER BY id", "[[2];[3];[4];[5]]" }, + { "SELECT id FROM `" + tableName + "` WHERE dyn >= CAST(\"0\" AS DyNumber) AND dyn <= CAST(\"10\" AS DyNumber) ORDER BY id", "[[2];[3];[4]]" }, + }, + }; } - void BulkUpsertRowTableArrow(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - auto batch = MakeArrowBatch(rows); - TString strBatch = NArrow::SerializeBatchNoCompression(batch); - TString strSchema = NArrow::SerializeSchema(*batch->schema()); - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, NYdb::NTable::EDataFormat::ApacheArrow, strBatch, strSchema).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + TScenario<TString> OrderByScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, MakeDyNumber("100") }, + { 2, 20, MakeDyNumber("-50") }, + { 3, 30, MakeDyNumber("0") }, + { 4, 40, MakeDyNumber("3.14") }, + { 5, 50, MakeDyNumber("-1") }, + }, + { + { "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn", "[[2;[\"-.5e2\"]];[5;[\"-.1e1\"]];[3;[\"0\"]];[4;[\".314e1\"]];[1;[\".1e3\"]]]" }, + { "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn DESC", "[[1;[\".1e3\"]];[4;[\".314e1\"]];[3;[\"0\"]];[5;[\"-.1e1\"]];[2;[\"-.5e2\"]]]" }, + }, + }; } - void LoadData(TTestHelper& helper, ETableKind table, ELoadKind load, const TString& name, const TVector<TRow>& rows, - TTestHelper::TColumnTable* col = nullptr, const TVector<TTestHelper::TColumnSchema>* schema = nullptr) { - switch (table) { - case ETableKind::COLUMNSHARD: { - Y_ABORT_UNLESS(col && schema); - if (load == ELoadKind::ARROW) { - auto batch = MakeArrowBatch(rows); - helper.BulkUpsert(*col, batch); - } else if (load == ELoadKind::YDB_VALUE) { - BulkUpsertRowTableYdbValue(helper, name, rows); - } else { - BulkUpsertRowTableCSV(helper, name, rows); - } + TScenario<TString> GroupByScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, MakeDyNumber("1") }, + { 2, 20, MakeDyNumber("2") }, + { 3, 30, MakeDyNumber("1") }, + { 4, 40, MakeDyNumber("2") }, + { 5, 50, MakeDyNumber("3") }, + }, + { + { "SELECT dyn, count(*) AS cnt FROM `" + tableName + "` GROUP BY dyn ORDER BY dyn", + "[[[\".1e1\"];2u];[[\".2e1\"];2u];[[\".3e1\"];1u]]" }, + }, + }; + } - break; - } - case ETableKind::DATASHARD: { - if (load == ELoadKind::ARROW) { - BulkUpsertRowTableArrow(helper, name, rows); - } else if (load == ELoadKind::YDB_VALUE) { - BulkUpsertRowTableYdbValue(helper, name, rows); - } else { - BulkUpsertRowTableCSV(helper, name, rows); - } + TScenario<TString> AggregationScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, MakeDyNumber("-5") }, + { 2, 20, MakeDyNumber("10") }, + { 3, 30, MakeDyNumber("99") }, + { 4, 40, std::nullopt }, + { 5, 50, MakeDyNumber("0") }, + }, + { + { "SELECT min(dyn) FROM `" + tableName + "`", "[[[\"-.5e1\"]]]" }, + { "SELECT max(dyn) FROM `" + tableName + "`", "[[[\".99e2\"]]]" }, + { "SELECT count(dyn) FROM `" + tableName + "`", "[[4u]]" }, + { "SELECT count(*) FROM `" + tableName + "`", "[[5u]]" }, + }, + }; + } - break; - } - } + TJoinScenario<TString> JoinByDyNumberScenario() { + const TString t1 = "/Root/Table1"; + const TString t2 = "/Root/Table2"; + return { + t1, + t2, + { + { 1, 10, MakeDyNumber("10") }, + { 2, 20, MakeDyNumber("20") }, + { 3, 30, MakeDyNumber("30") }, + }, + { + { 10, 100, MakeDyNumber("10") }, + { 20, 200, MakeDyNumber("30") }, + { 30, 300, MakeDyNumber("50") }, + }, + { + { "SELECT t1.id, t2.id, t1.dyn FROM `" + t1 + "` AS t1 " + "JOIN `" + t2 + "` AS t2 ON t1.dyn = t2.dyn " + "ORDER BY t1.id, t2.id", + "[[1;10;[\".1e2\"]];[3;20;[\".3e2\"]]]" }, + }, + }; } - void CheckOrExec(TTestHelper& helper, const TString& query, const TString& expected, EQueryMode scanMode) { - if (scanMode == EQueryMode::SCAN_QUERY) { - helper.ReadData(query, expected); - } else { - helper.ReadDataExecQuery(query, expected); - } + TScenario<TString> OrderByWithLimitScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, MakeDyNumber("-100") }, + { 2, 20, MakeDyNumber("-1") }, + { 3, 30, MakeDyNumber("0") }, + { 4, 40, MakeDyNumber("5") }, + { 5, 50, MakeDyNumber("42") }, + { 6, 60, MakeDyNumber("999") }, + }, + { + { "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn LIMIT 3", "[[1;[\"-.1e3\"]];[2;[\"-.1e1\"]];[3;[\"0\"]]]" }, + { "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn DESC LIMIT 2", "[[6;[\".999e3\"]];[5;[\".42e2\"]]]" }, + { "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn LIMIT 2 OFFSET 2", "[[3;[\"0\"]];[4;[\".5e1\"]]]" }, + }, + }; } - void PrepareBase(TTestHelper& helper, ETableKind tableKind, const TString& tableName, TTestHelper::TColumnTable* colTableOut, - TVector<TTestHelper::TColumnSchema>* schemaOut) { - if (tableKind == ETableKind::COLUMNSHARD) { - TVector<TTestHelper::TColumnSchema> schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("dyn").SetType(NScheme::NTypeIds::DyNumber), - }; + TScenario<TString> GroupByWithNullsScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { 1, 10, MakeDyNumber("10") }, + { 2, 20, std::nullopt }, + { 3, 30, MakeDyNumber("10") }, + { 4, 40, std::nullopt }, + { 5, 50, MakeDyNumber("20") }, + { 6, 60, std::nullopt }, + }, + { + { "SELECT dyn, count(*) AS cnt FROM `" + tableName + "` GROUP BY dyn ORDER BY dyn", + "[[#;3u];[[\".1e2\"];2u];[[\".2e2\"];1u]]" }, + { "SELECT count(dyn), count(*) FROM `" + tableName + "`", "[[3u;6u]]" }, + }, + }; + } - *schemaOut = schema; - TTestHelper::TColumnTable col; - col.SetName(tableName).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema); - helper.CreateTable(col); - *colTableOut = col; - } else { - CreateDataShardTable(helper, tableName); - } + TPkLookupScenario PkLookupScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { "SELECT val FROM `" + tableName + "` WHERE dyn = CAST(\".1e3\" AS DyNumber)", "[[[4]]]" }, + { "SELECT val FROM `" + tableName + "` WHERE dyn = CAST(\"0\" AS DyNumber)", "[[[2]]]" }, + { "SELECT val FROM `" + tableName + "` WHERE dyn = CAST(\"3.14\" AS DyNumber)", "[[[3]]]" }, + }, + }; } } // namespace @@ -211,11 +306,11 @@ Y_UNIT_TEST_SUITE(KqpDyNumberColumnShard) { const auto Load = Arg<2>(); const TString tableName = "/Root/ColumnTableTest"; - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); + TTestHelper helper(TDyNumberTraits::CreateSettings()); TTestHelper::TColumnTable col; TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { + Base::PrepareBase(helper, Table, tableName, &col, &schema); + Base::LoadData(helper, Table, Load, tableName, { { 1, 10, MakeDyNumber("3.14") }, { 2, 20, MakeDyNumber("-100") }, { 3, 30, std::nullopt }, @@ -238,186 +333,19 @@ Y_UNIT_TEST_SUITE(KqpDyNumberColumnShard) { CheckOrExec(helper, "SELECT dyn FROM `" + tableName + "` WHERE id=6", "[[[\"-.5e1\"]]]", Scan); } - Y_UNIT_TEST(TestFilterEqual, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/ColumnTableTest"; - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { - { 1, 10, MakeDyNumber("1") }, - { 2, 20, MakeDyNumber("2") }, - { 3, 30, MakeDyNumber("3.14") }, - { 4, 40, MakeDyNumber("100") } - }, &col, &schema); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn = CAST(\"3.14\" AS DyNumber)", - "[[3]]", Scan); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn = CAST(\"1\" AS DyNumber)", - "[[1]]", Scan); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn != CAST(\"3.14\" AS DyNumber) ORDER BY id", - "[[1];[2];[4]]", Scan); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn != CAST(\"100\" AS DyNumber) ORDER BY id", - "[[1];[2];[3]]", Scan); - } - - Y_UNIT_TEST(TestFilterNulls, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/ColumnTableTest"; - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { - { 1, 10, MakeDyNumber("1") }, - { 2, 20, std::nullopt }, - { 3, 30, MakeDyNumber("3") }, - { 4, 40, std::nullopt }, - { 5, 50, MakeDyNumber("0") } - }, &col, &schema); - - CheckOrExec(helper, "SELECT id FROM `" + tableName + "` WHERE dyn IS NULL ORDER BY id", - "[[2];[4]]", Scan); - CheckOrExec(helper, "SELECT id FROM `" + tableName + "` WHERE dyn IS NOT NULL ORDER BY id", - "[[1];[3];[5]]", Scan); - } - - Y_UNIT_TEST(TestFilterCompare, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/ColumnTableTest"; - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { - { 1, 10, MakeDyNumber("-10") }, - { 2, 20, MakeDyNumber("0") }, - { 3, 30, MakeDyNumber("5") }, - { 4, 40, MakeDyNumber("10") }, - { 5, 50, MakeDyNumber("100") } - }, &col, &schema); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn < CAST(\"5\" AS DyNumber) ORDER BY id", - "[[1];[2]]", Scan); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn > CAST(\"5\" AS DyNumber) ORDER BY id", - "[[4];[5]]", Scan); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn <= CAST(\"5\" AS DyNumber) ORDER BY id", - "[[1];[2];[3]]", Scan); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn >= CAST(\"10\" AS DyNumber) ORDER BY id", - "[[4];[5]]", Scan); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn > CAST(\"-10\" AS DyNumber) ORDER BY id", - "[[2];[3];[4];[5]]", Scan); - - CheckOrExec(helper, - "SELECT id FROM `" + tableName + "` WHERE dyn >= CAST(\"0\" AS DyNumber) AND dyn <= CAST(\"10\" AS DyNumber) ORDER BY id", - "[[2];[3];[4]]", Scan); - } - - Y_UNIT_TEST(TestOrderBy, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/ColumnTableTest"; - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { - { 1, 10, MakeDyNumber("100") }, - { 2, 20, MakeDyNumber("-50") }, - { 3, 30, MakeDyNumber("0") }, - { 4, 40, MakeDyNumber("3.14") }, - { 5, 50, MakeDyNumber("-1") } - }, &col, &schema); - - CheckOrExec(helper, - "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn", - "[[2;[\"-.5e2\"]];[5;[\"-.1e1\"]];[3;[\"0\"]];[4;[\".314e1\"]];[1;[\".1e3\"]]]", Scan); - - CheckOrExec(helper, - "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn DESC", - "[[1;[\".1e3\"]];[4;[\".314e1\"]];[3;[\"0\"]];[5;[\"-.1e1\"]];[2;[\"-.5e2\"]]]", Scan); - } - - Y_UNIT_TEST(TestGroupBy, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/ColumnTableTest"; - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { - { 1, 10, MakeDyNumber("1") }, - { 2, 20, MakeDyNumber("2") }, - { 3, 30, MakeDyNumber("1") }, - { 4, 40, MakeDyNumber("2") }, - { 5, 50, MakeDyNumber("3") } - }, &col, &schema); - - CheckOrExec(helper, "SELECT dyn, count(*) AS cnt FROM `" + tableName + "` GROUP BY dyn ORDER BY dyn", - "[[[\".1e1\"];2u];[[\".2e1\"];2u];[[\".3e1\"];1u]]", Scan); - } - - Y_UNIT_TEST(TestAggregation, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/ColumnTableTest"; - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { - { 1, 10, MakeDyNumber("-5") }, - { 2, 20, MakeDyNumber("10") }, - { 3, 30, MakeDyNumber("99") }, - { 4, 40, std::nullopt }, - { 5, 50, MakeDyNumber("0") } - }, &col, &schema); - - CheckOrExec(helper, "SELECT min(dyn) FROM `" + tableName + "`", "[[[\"-.5e1\"]]]", Scan); - CheckOrExec(helper, "SELECT max(dyn) FROM `" + tableName + "`", "[[[\".99e2\"]]]", Scan); - CheckOrExec(helper, "SELECT count(dyn) FROM `" + tableName + "`", "[[4u]]", Scan); - CheckOrExec(helper, "SELECT count(*) FROM `" + tableName + "`", "[[5u]]", Scan); - } + Y_UNIT_TEST_SCENARIO(TestFilterEqual, FilterEqualScenario); + Y_UNIT_TEST_SCENARIO(TestFilterNulls, FilterNullsScenario); + Y_UNIT_TEST_SCENARIO(TestFilterCompare, FilterCompareScenario); + Y_UNIT_TEST_SCENARIO(TestOrderBy, OrderByScenario); + Y_UNIT_TEST_SCENARIO(TestGroupBy, GroupByScenario); + Y_UNIT_TEST_SCENARIO(TestAggregation, AggregationScenario); Y_UNIT_TEST(TestJoinById, EQueryMode, ETableKind, ELoadKind) { const auto Scan = Arg<0>(); const auto Table = Arg<1>(); const auto Load = Arg<2>(); - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); + TTestHelper helper(TDyNumberTraits::CreateSettings()); const TString t1 = "/Root/Table1"; const TString t2 = "/Root/Table2"; @@ -444,7 +372,7 @@ Y_UNIT_TEST_SUITE(KqpDyNumberColumnShard) { col2.SetName(t2).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s2); helper.CreateTable(col2); } else { - CreateDataShardTable(helper, t1); + Base::CreateDataShardTable(helper, t1); { auto& session = helper.GetSession(); auto res = session @@ -461,7 +389,7 @@ Y_UNIT_TEST_SUITE(KqpDyNumberColumnShard) { } } - LoadData(helper, Table, Load, t1, { + Base::LoadData(helper, Table, Load, t1, { { 1, 10, MakeDyNumber("3.14") }, { 2, 20, MakeDyNumber("100") } }, &col1, &s1); @@ -546,189 +474,23 @@ Y_UNIT_TEST_SUITE(KqpDyNumberColumnShard) { "[[1;[\".314e1\"];[\".5e2\"]];[1;[\".314e1\"];[\".6e2\"]];[2;[\".1e3\"];[\".7e2\"]]]", Scan); } - Y_UNIT_TEST(TestJoinByDyNumber, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - - const TString t1 = "/Root/Table1"; - const TString t2 = "/Root/Table2"; - - TTestHelper::TColumnTable col1, col2; - TVector<TTestHelper::TColumnSchema> s1, s2; - - if (Table == ETableKind::COLUMNSHARD) { - s1 = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("dyn").SetType(NScheme::NTypeIds::DyNumber), - }; - - col1.SetName(t1).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s1); - helper.CreateTable(col1); - - s2 = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("dyn").SetType(NScheme::NTypeIds::DyNumber), - }; - - col2.SetName(t2).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s2); - helper.CreateTable(col2); - } else { - CreateDataShardTable(helper, t1); - CreateDataShardTable(helper, t2); - } - - LoadData(helper, Table, Load, t1, { - { 1, 10, MakeDyNumber("10") }, - { 2, 20, MakeDyNumber("20") }, - { 3, 30, MakeDyNumber("30") } - }, &col1, &s1); - - LoadData(helper, Table, Load, t2, { - { 10, 100, MakeDyNumber("10") }, - { 20, 200, MakeDyNumber("30") }, - { 30, 300, MakeDyNumber("50") } - }, &col2, &s2); - - CheckOrExec(helper, - "SELECT t1.id, t2.id, t1.dyn FROM `" + t1 + "` AS t1 " - "JOIN `" + t2 + "` AS t2 ON t1.dyn = t2.dyn " - "ORDER BY t1.id, t2.id", - "[[1;10;[\".1e2\"]];[3;20;[\".3e2\"]]]", Scan); - } - - Y_UNIT_TEST(TestOrderByWithLimit, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/ColumnTableTest"; - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { - { 1, 10, MakeDyNumber("-100") }, - { 2, 20, MakeDyNumber("-1") }, - { 3, 30, MakeDyNumber("0") }, - { 4, 40, MakeDyNumber("5") }, - { 5, 50, MakeDyNumber("42") }, - { 6, 60, MakeDyNumber("999") } - }, &col, &schema); - - CheckOrExec(helper, - "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn LIMIT 3", - "[[1;[\"-.1e3\"]];[2;[\"-.1e1\"]];[3;[\"0\"]]]", Scan); - - CheckOrExec(helper, - "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn DESC LIMIT 2", - "[[6;[\".999e3\"]];[5;[\".42e2\"]]]", Scan); - - CheckOrExec(helper, - "SELECT id, dyn FROM `" + tableName + "` ORDER BY dyn LIMIT 2 OFFSET 2", - "[[3;[\"0\"]];[4;[\".5e1\"]]]", Scan); - } - - Y_UNIT_TEST(TestGroupByWithNulls, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/ColumnTableTest"; - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, { - { 1, 10, MakeDyNumber("10") }, - { 2, 20, std::nullopt }, - { 3, 30, MakeDyNumber("10") }, - { 4, 40, std::nullopt }, - { 5, 50, MakeDyNumber("20") }, - { 6, 60, std::nullopt } - }, &col, &schema); - - CheckOrExec(helper, - "SELECT dyn, count(*) AS cnt FROM `" + tableName + "` GROUP BY dyn ORDER BY dyn", - "[[#;3u];[[\".1e2\"];2u];[[\".2e2\"];1u]]", Scan); - - CheckOrExec(helper, - "SELECT count(dyn), count(*) FROM `" + tableName + "`", - "[[3u;6u]]", Scan); - } - - Y_UNIT_TEST(TestDyNumberAsPrimaryKey, EQueryMode, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Load = Arg<1>(); - - TTestHelper helper(CreateKikimrSettingsWithDyNumberSupport()); - - TVector<TTestHelper::TColumnSchema> schema = { - TTestHelper::TColumnSchema().SetName("dyn").SetType(NScheme::NTypeIds::DyNumber).SetNullable(false), - TTestHelper::TColumnSchema().SetName("val").SetType(NScheme::NTypeIds::Int64) - }; - - TTestHelper::TColumnTable testTable; - testTable.SetName("/Root/ColumnTableTest").SetPrimaryKey({"dyn"}).SetSharding({"dyn"}).SetSchema(schema); - helper.CreateTable(testTable); - - if (Load == ELoadKind::ARROW) { - arrow::BinaryBuilder dynB; - arrow::Int64Builder valB; - AppendDyNumber(dynB, "-100"); Y_ABORT_UNLESS(valB.Append(1).ok()); - AppendDyNumber(dynB, "0"); Y_ABORT_UNLESS(valB.Append(2).ok()); - AppendDyNumber(dynB, "3.14"); Y_ABORT_UNLESS(valB.Append(3).ok()); - AppendDyNumber(dynB, "100"); Y_ABORT_UNLESS(valB.Append(4).ok()); - AppendDyNumber(dynB, "999"); Y_ABORT_UNLESS(valB.Append(5).ok()); - std::shared_ptr<arrow::Array> dynArr, valArr; - Y_ABORT_UNLESS(dynB.Finish(&dynArr).ok()); - Y_ABORT_UNLESS(valB.Finish(&valArr).ok()); - auto aSchema = arrow::schema({ - arrow::field("dyn", arrow::binary(), false), - arrow::field("val", arrow::int64()) - }); - - auto batch = arrow::RecordBatch::Make(aSchema, 5, { dynArr, valArr }); - helper.BulkUpsert(testTable, batch); - } else if (Load == ELoadKind::YDB_VALUE) { - TValueBuilder builder; - builder.BeginList(); - builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("-100").AddMember("val").BeginOptional().Int64(1).EndOptional().EndStruct(); - builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("0").AddMember("val").BeginOptional().Int64(2).EndOptional().EndStruct(); - builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("3.14").AddMember("val").BeginOptional().Int64(3).EndOptional().EndStruct(); - builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("100").AddMember("val").BeginOptional().Int64(4).EndOptional().EndStruct(); - builder.AddListItem().BeginStruct().AddMember("dyn").DyNumber("999").AddMember("val").BeginOptional().Int64(5).EndOptional().EndStruct(); - builder.EndList(); - auto res = helper.GetKikimr().GetTableClient().BulkUpsert("/Root/ColumnTableTest", builder.Build()).GetValueSync(); - UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); - } else { - TStringBuilder csv; - csv << "-.1e3,1\n.0,2\n.314e1,3\n.1e3,4\n.999e3,5\n"; - auto res = helper.GetKikimr().GetTableClient().BulkUpsert("/Root/ColumnTableTest", EDataFormat::CSV, csv).GetValueSync(); - UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); - } - - CheckOrExec(helper, "SELECT val FROM `/Root/ColumnTableTest` WHERE dyn = CAST(\".1e3\" AS DyNumber)", "[[[4]]]", Scan); - CheckOrExec(helper, "SELECT val FROM `/Root/ColumnTableTest` WHERE dyn = CAST(\"0\" AS DyNumber)", "[[[2]]]", Scan); - CheckOrExec(helper, "SELECT val FROM `/Root/ColumnTableTest` WHERE dyn = CAST(\"3.14\" AS DyNumber)", "[[[3]]]", Scan); - } + Y_UNIT_TEST_JOIN_SCENARIO(TestJoinByDyNumber, JoinByDyNumberScenario); + Y_UNIT_TEST_SCENARIO(TestOrderByWithLimit, OrderByWithLimitScenario); + Y_UNIT_TEST_SCENARIO(TestGroupByWithNulls, GroupByWithNullsScenario); + Y_UNIT_TEST_PK_SCENARIO(TestDyNumberAsPrimaryKey, PkLookupScenario); Y_UNIT_TEST(TestDmlParityAndCTAS, EQueryMode, ELoadKind) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - auto runnerSettings = CreateKikimrSettingsWithDyNumberSupport(); + auto runnerSettings = TDyNumberTraits::CreateSettings(); runnerSettings.AppConfig.MutableTableServiceConfig()->SetEnableHtapTx(true); TTestHelper helper(runnerSettings); const TString ds = "/Root/RowSrc"; const TString cs = "/Root/ColSrc"; - CreateDataShardTable(helper, ds); + Base::CreateDataShardTable(helper, ds); TVector<TTestHelper::TColumnSchema> schema = { TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), @@ -746,19 +508,19 @@ Y_UNIT_TEST_SUITE(KqpDyNumberColumnShard) { "(2, 200, CAST(\"20\" AS DyNumber))"); if (Load == ELoadKind::ARROW) { - auto batch = MakeArrowBatch({ + auto batch = Base::MakeArrowBatch({ { 1, 100, MakeDyNumber("10") }, { 2, 200, MakeDyNumber("20") } }); helper.BulkUpsert(col, batch); } else if (Load == ELoadKind::YDB_VALUE) { - BulkUpsertRowTableYdbValue(helper, cs, { + Base::BulkUpsertRowTableYdbValue(helper, cs, { { 1, 100, MakeDyNumber("10") }, { 2, 200, MakeDyNumber("20") } }); } else { - BulkUpsertRowTableCSV(helper, cs, { + Base::BulkUpsertRowTableCSV(helper, cs, { { 1, 100, MakeDyNumber("10") }, { 2, 200, MakeDyNumber("20") } }); diff --git a/ydb/core/kqp/ut/olap/types/interval_ut.cpp b/ydb/core/kqp/ut/olap/types/interval_ut.cpp index 47a3805aa1a..b3c1c279f8b 100644 --- a/ydb/core/kqp/ut/olap/types/interval_ut.cpp +++ b/ydb/core/kqp/ut/olap/types/interval_ut.cpp @@ -1,22 +1,5 @@ -#include "bool_test_enums.h" - -#include <ydb/core/formats/arrow/arrow_helpers.h> -#include <ydb/core/kqp/ut/common/columnshard.h> -#include <ydb/core/kqp/ut/common/kqp_ut_common.h> -#include <ydb/core/testlib/common_helper.h> -#include <ydb/core/testlib/cs_helper.h> -#include <ydb/core/tx/columnshard/hooks/testing/controller.h> -#include <ydb/core/tx/tx_proxy/proxy.h> - -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_replication.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/client.h> - -#include <library/cpp/threading/local_executor/local_executor.h> -#include <util/generic/serialized_enum.h> -#include <util/string/printf.h> -#include <ydb/core/kqp/ut/common/arrow_builders.h> +#include "column_type_scenarios.h" +#include "column_type_test_base.h" namespace NKikimr { namespace NKqp { @@ -26,163 +9,43 @@ using namespace NYdb::NTable; Y_UNIT_TEST_SUITE(KqpIntervalColumnShard) { namespace { - struct TRow { - i32 Id; - i64 IntVal; - std::optional<i64> Ival; - }; - - constexpr i64 MaxValidInterval = (i64)(86400000000ULL * 49673U - 1); - constexpr i64 MinValidInterval = -MaxValidInterval; - - TKikimrSettings CreateKikimrSettingsWithIntervalSupport() { - NKikimrConfig::TFeatureFlags featureFlags; - featureFlags.SetEnableColumnshardInterval(true); - return TKikimrSettings().SetWithSampleTables(false).SetFeatureFlags(featureFlags); - } - void CreateDataShardTable(TTestHelper& helper, const TString& name) { - auto& session = helper.GetSession(); - auto res = session - .ExecuteSchemeQuery(TStringBuilder() << R"( - CREATE TABLE `)" << name << R"(` ( - id Int32 NOT NULL, - int Int64, - ival Interval, - PRIMARY KEY (id) - ); - )") - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), NYdb::EStatus::SUCCESS); - } - - void BulkUpsertRowTableYdbValue(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - TValueBuilder builder; - builder.BeginList(); - for (auto&& r : rows) { - builder.AddListItem().BeginStruct() - .AddMember("id").Int32(r.Id) - .AddMember("int").Int64(r.IntVal) - .AddMember("ival"); - if (r.Ival.has_value()) { - builder.BeginOptional().Interval(*r.Ival).EndOptional(); + struct TIntervalTraits { + using TValue = i64; + static constexpr const char* ColumnName = "ival"; + static constexpr const char* SqlTypeName = "Interval"; + static TKikimrSettings CreateSettings() { + return CreateColumnshardSettings([](auto& f) { f.SetEnableColumnshardInterval(true); }); + } + static auto GetTypeId() { return NScheme::NTypeIds::Interval; } + static void AppendYdbValue(TValueBuilder& builder, const std::optional<i64>& val) { + if (val.has_value()) { + builder.BeginOptional().Interval(*val).EndOptional(); } else { builder.EmptyOptional(EPrimitiveType::Interval); } - - builder.EndStruct(); } - - builder.EndList(); - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, builder.Build()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void BulkUpsertRowTableCSV(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - TStringBuilder builder; - for (auto&& r : rows) { - builder << r.Id << "," << r.IntVal << ","; - if (r.Ival.has_value()) { - builder << *r.Ival; + static void AppendCsvValue(TStringBuilder& builder, const std::optional<i64>& val) { + if (val.has_value()) { + builder << *val; } - - builder << '\n'; } - - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, EDataFormat::CSV, builder).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - std::shared_ptr<arrow::RecordBatch> MakeArrowBatch(const TVector<TRow>& rows) { - using namespace NKikimr::NKqp::NTestArrow; - std::vector<int32_t> ids; - std::vector<int64_t> ints; - std::vector<std::optional<int64_t>> ivals; - ids.reserve(rows.size()); - ints.reserve(rows.size()); - ivals.reserve(rows.size()); - for (auto&& r : rows) { - ids.push_back(r.Id); - ints.push_back(r.IntVal); - ivals.push_back(r.Ival); - } - - auto idArr = MakeInt32Array(ids); - auto intArr = MakeInt64Array(ints); - auto ivalArr = MakeInt64ArrayNullable(ivals); - auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), /*nullable*/ false), - arrow::field("int", arrow::int64()), - arrow::field("ival", arrow::int64()) - }); - - return MakeBatch({ schema->field(0), schema->field(1), schema->field(2) }, { idArr, intArr, ivalArr }); - } - - void BulkUpsertRowTableArrow(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - auto batch = MakeArrowBatch(rows); - TString strBatch = NArrow::SerializeBatchNoCompression(batch); - TString strSchema = NArrow::SerializeSchema(*batch->schema()); - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, NYdb::NTable::EDataFormat::ApacheArrow, strBatch, strSchema).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void LoadData(TTestHelper& helper, ETableKind table, ELoadKind load, const TString& name, const TVector<TRow>& rows, - TTestHelper::TColumnTable* col = nullptr, const TVector<TTestHelper::TColumnSchema>* schema = nullptr) { - switch (table) { - case ETableKind::COLUMNSHARD: { - Y_ABORT_UNLESS(col && schema); - if (load == ELoadKind::ARROW) { - auto batch = MakeArrowBatch(rows); - helper.BulkUpsert(*col, batch); - } else if (load == ELoadKind::YDB_VALUE) { - BulkUpsertRowTableYdbValue(helper, name, rows); - } else { - BulkUpsertRowTableCSV(helper, name, rows); - } - - break; - } - case ETableKind::DATASHARD: { - if (load == ELoadKind::ARROW) { - BulkUpsertRowTableArrow(helper, name, rows); - } else if (load == ELoadKind::YDB_VALUE) { - BulkUpsertRowTableYdbValue(helper, name, rows); - } else { - BulkUpsertRowTableCSV(helper, name, rows); - } - - break; + static std::shared_ptr<arrow::Array> MakeArrowArray(const TVector<TTypedRow<i64>>& rows) { + using namespace NKikimr::NKqp::NTestArrow; + std::vector<std::optional<int64_t>> vals; + vals.reserve(rows.size()); + for (auto&& r : rows) { + vals.push_back(r.TypedVal); } + return MakeInt64ArrayNullable(vals); } - } - - void CheckOrExec(TTestHelper& helper, const TString& query, const TString& expected, EQueryMode scanMode) { - if (scanMode == EQueryMode::SCAN_QUERY) { - helper.ReadData(query, expected); - } else { - helper.ReadDataExecQuery(query, expected); - } - } + static std::shared_ptr<arrow::DataType> ArrowType() { return arrow::int64(); } + }; - void PrepareBase(TTestHelper& helper, ETableKind tableKind, const TString& tableName, TTestHelper::TColumnTable* colTableOut, - TVector<TTestHelper::TColumnSchema>* schemaOut) { - if (tableKind == ETableKind::COLUMNSHARD) { - TVector<TTestHelper::TColumnSchema> schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("ival").SetType(NScheme::NTypeIds::Interval), - }; + COLUMN_TYPE_TEST_USING(TIntervalTraits); - *schemaOut = schema; - TTestHelper::TColumnTable col; - col.SetName(tableName).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema); - helper.CreateTable(col); - *colTableOut = col; - } else { - CreateDataShardTable(helper, tableName); - } - } + constexpr i64 MaxValidInterval = (i64)(86400000000ULL * 49673U - 1); + constexpr i64 MinValidInterval = -MaxValidInterval; void CreateIntervalPkDataShardTable(TTestHelper& helper, const TString& name) { auto& session = helper.GetSession(); @@ -253,232 +116,217 @@ Y_UNIT_TEST_SUITE(KqpIntervalColumnShard) { UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); } } - } // namespace - - Y_UNIT_TEST(TestSimpleQueries, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 10, (i64)1000000 }, { 2, 20, (i64)-500000 }, { 3, 30, std::nullopt }, { 4, 40, (i64)0 }, - { 5, 50, MaxValidInterval }, { 6, 60, MinValidInterval } }, - &col, &schema); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=1", "[[1;[10];[1000000]]]", Scan); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=2", "[[2;[20];[-500000]]]", Scan); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=3", "[[3;[30];#]]", Scan); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=4", "[[4;[40];[0]]]", Scan); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=5", TStringBuilder() << "[[5;[50];[" << MaxValidInterval << "]]]", Scan); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=6", TStringBuilder() << "[[6;[60];[" << MinValidInterval << "]]]", Scan); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY id", - TStringBuilder() << "[[1;[10];[1000000]];[2;[20];[-500000]];[3;[30];#];[4;[40];[0]];[5;[50];[" << MaxValidInterval - << "]];[6;[60];[" << MinValidInterval << "]]]", Scan); - - CheckOrExec(helper, "SELECT ival FROM `/Root/Table1` WHERE id=1", "[[[1000000]]]", Scan); - CheckOrExec(helper, "SELECT ival FROM `/Root/Table1` WHERE id=3", "[[#]]", Scan); - - LoadData(helper, Table, Load, tableName, { { 7, 0, std::nullopt } }, &col, &schema); - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=7", "[[7;[0];#]]", Scan); - } - Y_UNIT_TEST(TestFilterEqual, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, + TScenario<i64> FilterEqualScenario() { + return { + "/Root/Table1", { { 1, 10, (i64)1000000 }, { 2, 20, (i64)2000000 }, { 3, 30, (i64)1000000 }, { 4, 40, std::nullopt }, { 5, 50, (i64)-500000 } }, - &col, &schema); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival = CAST(1000000 AS Interval) ORDER BY id", - "[[1;[10];[1000000]];[3;[30];[1000000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival = CAST(2000000 AS Interval) ORDER BY id", - "[[2;[20];[2000000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival = CAST(-500000 AS Interval) ORDER BY id", - "[[5;[50];[-500000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival = CAST(999 AS Interval)", "[]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival != CAST(1000000 AS Interval) ORDER BY id", - "[[2;[20];[2000000]];[5;[50];[-500000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival != CAST(-500000 AS Interval) ORDER BY id", - "[[1;[10];[1000000]];[2;[20];[2000000]];[3;[30];[1000000]]]", Scan); + { + { "SELECT * FROM `/Root/Table1` WHERE ival = CAST(1000000 AS Interval) ORDER BY id", "[[1;[10];[1000000]];[3;[30];[1000000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival = CAST(2000000 AS Interval) ORDER BY id", "[[2;[20];[2000000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival = CAST(-500000 AS Interval) ORDER BY id", "[[5;[50];[-500000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival = CAST(999 AS Interval)", "[]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival != CAST(1000000 AS Interval) ORDER BY id", "[[2;[20];[2000000]];[5;[50];[-500000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival != CAST(-500000 AS Interval) ORDER BY id", "[[1;[10];[1000000]];[2;[20];[2000000]];[3;[30];[1000000]]]" }, + }, + }; } - Y_UNIT_TEST(TestFilterNulls, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, + TScenario<i64> FilterNullsScenario() { + return { + "/Root/Table1", { { 1, 10, (i64)100 }, { 2, 20, std::nullopt }, { 3, 30, (i64)300 }, { 4, 40, std::nullopt }, { 5, 50, (i64)0 }, { 6, 60, std::nullopt } }, - &col, &schema); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival IS NULL ORDER BY id", - "[[2;[20];#];[4;[40];#];[6;[60];#]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival IS NOT NULL ORDER BY id", - "[[1;[10];[100]];[3;[30];[300]];[5;[50];[0]]]", Scan); - - CheckOrExec(helper, "SELECT count(*) FROM `/Root/Table1` WHERE ival IS NULL", "[[3u]]", Scan); - CheckOrExec(helper, "SELECT count(*) FROM `/Root/Table1` WHERE ival IS NOT NULL", "[[3u]]", Scan); + { + { "SELECT * FROM `/Root/Table1` WHERE ival IS NULL ORDER BY id", "[[2;[20];#];[4;[40];#];[6;[60];#]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival IS NOT NULL ORDER BY id", "[[1;[10];[100]];[3;[30];[300]];[5;[50];[0]]]" }, + { "SELECT count(*) FROM `/Root/Table1` WHERE ival IS NULL", "[[3u]]" }, + { "SELECT count(*) FROM `/Root/Table1` WHERE ival IS NOT NULL", "[[3u]]" }, + }, + }; } - Y_UNIT_TEST(TestFilterCompare, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, + TScenario<i64> FilterCompareScenario() { + return { + "/Root/Table1", { { 1, 10, (i64)-3000000 }, { 2, 20, (i64)-1000000 }, { 3, 30, (i64)0 }, { 4, 40, (i64)1000000 }, { 5, 50, (i64)5000000 }, { 6, 60, std::nullopt } }, - &col, &schema); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival < CAST(0 AS Interval) ORDER BY id", - "[[1;[10];[-3000000]];[2;[20];[-1000000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival > CAST(0 AS Interval) ORDER BY id", - "[[4;[40];[1000000]];[5;[50];[5000000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival <= CAST(0 AS Interval) ORDER BY id", - "[[1;[10];[-3000000]];[2;[20];[-1000000]];[3;[30];[0]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival >= CAST(0 AS Interval) ORDER BY id", - "[[3;[30];[0]];[4;[40];[1000000]];[5;[50];[5000000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival >= CAST(-1000000 AS Interval) AND ival <= CAST(1000000 AS Interval) ORDER BY id", - "[[2;[20];[-1000000]];[3;[30];[0]];[4;[40];[1000000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival > CAST(-1000000 AS Interval) AND ival < CAST(5000000 AS Interval) ORDER BY id", - "[[3;[30];[0]];[4;[40];[1000000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival < CAST(-1000000 AS Interval) ORDER BY id", - "[[1;[10];[-3000000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE ival < CAST(99999999 AS Interval) ORDER BY id", - "[[1;[10];[-3000000]];[2;[20];[-1000000]];[3;[30];[0]];[4;[40];[1000000]];[5;[50];[5000000]]]", Scan); + { + { "SELECT * FROM `/Root/Table1` WHERE ival < CAST(0 AS Interval) ORDER BY id", "[[1;[10];[-3000000]];[2;[20];[-1000000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival > CAST(0 AS Interval) ORDER BY id", "[[4;[40];[1000000]];[5;[50];[5000000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival <= CAST(0 AS Interval) ORDER BY id", "[[1;[10];[-3000000]];[2;[20];[-1000000]];[3;[30];[0]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival >= CAST(0 AS Interval) ORDER BY id", "[[3;[30];[0]];[4;[40];[1000000]];[5;[50];[5000000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival >= CAST(-1000000 AS Interval) AND ival <= CAST(1000000 AS Interval) ORDER BY id", + "[[2;[20];[-1000000]];[3;[30];[0]];[4;[40];[1000000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival > CAST(-1000000 AS Interval) AND ival < CAST(5000000 AS Interval) ORDER BY id", + "[[3;[30];[0]];[4;[40];[1000000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival < CAST(-1000000 AS Interval) ORDER BY id", "[[1;[10];[-3000000]]]" }, + { "SELECT * FROM `/Root/Table1` WHERE ival < CAST(99999999 AS Interval) ORDER BY id", + "[[1;[10];[-3000000]];[2;[20];[-1000000]];[3;[30];[0]];[4;[40];[1000000]];[5;[50];[5000000]]]" }, + }, + }; } - Y_UNIT_TEST(TestOrderByInterval, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, + TScenario<i64> OrderByScenario() { + return { + "/Root/Table1", { { 1, 10, (i64)300 }, { 2, 20, (i64)-100 }, { 3, 30, (i64)200 }, { 4, 40, (i64)-200 }, { 5, 50, (i64)0 } }, - &col, &schema); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival", - "[[4;[40];[-200]];[2;[20];[-100]];[5;[50];[0]];[3;[30];[200]];[1;[10];[300]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival DESC", - "[[1;[10];[300]];[3;[30];[200]];[5;[50];[0]];[2;[20];[-100]];[4;[40];[-200]]]", Scan); - - LoadData(helper, Table, Load, tableName, { { 6, 60, (i64)200 } }, &col, &schema); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival, id", - "[[4;[40];[-200]];[2;[20];[-100]];[5;[50];[0]];[3;[30];[200]];[6;[60];[200]];[1;[10];[300]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival DESC, id DESC", - "[[1;[10];[300]];[6;[60];[200]];[3;[30];[200]];[5;[50];[0]];[2;[20];[-100]];[4;[40];[-200]]]", Scan); + { + { "SELECT * FROM `/Root/Table1` ORDER BY ival", "[[4;[40];[-200]];[2;[20];[-100]];[5;[50];[0]];[3;[30];[200]];[1;[10];[300]]]" }, + { "SELECT * FROM `/Root/Table1` ORDER BY ival DESC", "[[1;[10];[300]];[3;[30];[200]];[5;[50];[0]];[2;[20];[-100]];[4;[40];[-200]]]" }, + }, + { + { + { { 6, 60, (i64)200 } }, + { + { "SELECT * FROM `/Root/Table1` ORDER BY ival, id", + "[[4;[40];[-200]];[2;[20];[-100]];[5;[50];[0]];[3;[30];[200]];[6;[60];[200]];[1;[10];[300]]]" }, + { "SELECT * FROM `/Root/Table1` ORDER BY ival DESC, id DESC", + "[[1;[10];[300]];[6;[60];[200]];[3;[30];[200]];[5;[50];[0]];[2;[20];[-100]];[4;[40];[-200]]]" }, + }, + }, + }, + }; } - Y_UNIT_TEST(TestGroupByInterval, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, + TScenario<i64> GroupByScenario() { + return { + "/Root/Table1", { { 1, 10, (i64)100 }, { 2, 20, (i64)200 }, { 3, 30, (i64)100 }, { 4, 40, (i64)200 }, { 5, 50, (i64)300 }, { 6, 60, (i64)100 }, { 7, 70, (i64)-100 } }, - &col, &schema); + { + { "SELECT ival, count(*) AS cnt FROM `/Root/Table1` GROUP BY ival ORDER BY ival", + "[[[-100];1u];[[100];3u];[[200];2u];[[300];1u]]" }, + { "SELECT ival, count(*) AS cnt, min(id) AS min_id FROM `/Root/Table1` GROUP BY ival ORDER BY ival", + "[[[-100];1u;7];[[100];3u;1];[[200];2u;2];[[300];1u;5]]" }, + }, + }; + } - CheckOrExec(helper, "SELECT ival, count(*) AS cnt FROM `/Root/Table1` GROUP BY ival ORDER BY ival", - "[[[-100];1u];[[100];3u];[[200];2u];[[300];1u]]", Scan); + TScenario<i64> AggregationScenario() { + return { + "/Root/Table1", + { { 1, 10, (i64)-500 }, { 2, 20, (i64)100 }, { 3, 30, (i64)999 }, + { 4, 40, std::nullopt }, { 5, 50, (i64)0 } }, + { + { "SELECT min(ival) FROM `/Root/Table1`", "[[[-500]]]" }, + { "SELECT max(ival) FROM `/Root/Table1`", "[[[999]]]" }, + { "SELECT count(ival) FROM `/Root/Table1`", "[[4u]]" }, + { "SELECT count(*) FROM `/Root/Table1`", "[[5u]]" }, + { "SELECT min(ival), max(ival), count(ival), count(*) FROM `/Root/Table1`", "[[[-500];[999];4u;5u]]" }, + }, + }; + } - CheckOrExec(helper, "SELECT ival, count(*) AS cnt, min(id) AS min_id FROM `/Root/Table1` GROUP BY ival ORDER BY ival", - "[[[-100];1u;7];[[100];3u;1];[[200];2u;2];[[300];1u;5]]", Scan); + TScenario<i64> AggregationAllNullScenario() { + return { + "/Root/Table2", + { { 1, 0, std::nullopt }, { 2, 0, std::nullopt } }, + { + { "SELECT min(ival), max(ival), count(ival), count(*) FROM `/Root/Table2`", "[[#;#;0u;2u]]" }, + }, + }; } - Y_UNIT_TEST(TestAggregation, EQueryMode, ETableKind, ELoadKind) { + TJoinScenario<i64> JoinByIntervalScenario() { + return { + "/Root/JoinTable1", + "/Root/JoinTable2", + { { 1, 10, (i64)1000 }, { 2, 20, (i64)2000 }, { 3, 30, (i64)3000 } }, + { { 10, 100, (i64)1000 }, { 20, 200, (i64)2000 }, { 30, 300, (i64)4000 } }, + { + { "SELECT t1.id, t2.id, t1.ival FROM `/Root/JoinTable1` AS t1 " + "JOIN `/Root/JoinTable2` AS t2 ON t1.ival = t2.ival " + "ORDER BY t1.id, t2.id", + "[[1;10;[1000]];[2;20;[2000]]]" }, + }, + }; + } + + TScenario<i64> OrderByWithLimitScenario() { + return { + "/Root/Table1", + { { 1, 10, (i64)500 }, { 2, 20, (i64)-200 }, { 3, 30, (i64)100 }, + { 4, 40, (i64)-300 }, { 5, 50, (i64)1000 }, { 6, 60, (i64)0 } }, + { + { "SELECT * FROM `/Root/Table1` ORDER BY ival LIMIT 1", "[[4;[40];[-300]]]" }, + { "SELECT * FROM `/Root/Table1` ORDER BY ival DESC LIMIT 1", "[[5;[50];[1000]]]" }, + { "SELECT * FROM `/Root/Table1` ORDER BY ival LIMIT 3", "[[4;[40];[-300]];[2;[20];[-200]];[6;[60];[0]]]" }, + { "SELECT * FROM `/Root/Table1` ORDER BY ival DESC LIMIT 3", "[[5;[50];[1000]];[1;[10];[500]];[3;[30];[100]]]" }, + { "SELECT * FROM `/Root/Table1` ORDER BY ival LIMIT 2 OFFSET 2", "[[6;[60];[0]];[3;[30];[100]]]" }, + { "SELECT * FROM `/Root/Table1` ORDER BY ival LIMIT 100", + "[[4;[40];[-300]];[2;[20];[-200]];[6;[60];[0]];[3;[30];[100]];[1;[10];[500]];[5;[50];[1000]]]" }, + { "SELECT * FROM `/Root/Table1` ORDER BY ival, id LIMIT 2", "[[4;[40];[-300]];[2;[20];[-200]]]" }, + }, + }; + } + + TScenario<i64> GroupByWithNullsScenario() { + return { + "/Root/Table1", + { { 1, 10, (i64)100 }, { 2, 20, (i64)200 }, { 3, 30, std::nullopt }, + { 4, 40, (i64)100 }, { 5, 50, std::nullopt }, { 6, 60, (i64)200 }, + { 7, 70, std::nullopt }, { 8, 80, (i64)-100 } }, + { + { "SELECT ival, count(*) AS cnt FROM `/Root/Table1` GROUP BY ival ORDER BY ival", + "[[#;3u];[[-100];1u];[[100];2u];[[200];2u]]" }, + { "SELECT ival, count(ival) AS cnt FROM `/Root/Table1` GROUP BY ival ORDER BY ival", + "[[#;0u];[[-100];1u];[[100];2u];[[200];2u]]" }, + { "SELECT ival, count(*) AS cnt FROM `/Root/Table1` WHERE ival IS NOT NULL GROUP BY ival ORDER BY ival", + "[[[-100];1u];[[100];2u];[[200];2u]]" }, + { "SELECT count(*) FROM `/Root/Table1` WHERE ival IS NULL", "[[3u]]" }, + { "SELECT min(ival), max(ival) FROM `/Root/Table1`", "[[[-100];[200]]]" }, + }, + }; + } + + } // namespace + + Y_UNIT_TEST(TestSimpleQueries, EQueryMode, ETableKind, ELoadKind) { const auto Scan = Arg<0>(); const auto Table = Arg<1>(); const auto Load = Arg<2>(); const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); + TTestHelper helper(TIntervalTraits::CreateSettings()); TTestHelper::TColumnTable col; TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 10, (i64)-500 }, { 2, 20, (i64)100 }, { 3, 30, (i64)999 }, - { 4, 40, std::nullopt }, { 5, 50, (i64)0 } }, + Base::PrepareBase(helper, Table, tableName, &col, &schema); + Base::LoadData(helper, Table, Load, tableName, + { { 1, 10, (i64)1000000 }, { 2, 20, (i64)-500000 }, { 3, 30, std::nullopt }, { 4, 40, (i64)0 }, + { 5, 50, MaxValidInterval }, { 6, 60, MinValidInterval } }, &col, &schema); - CheckOrExec(helper, "SELECT min(ival) FROM `/Root/Table1`", "[[[-500]]]", Scan); - CheckOrExec(helper, "SELECT max(ival) FROM `/Root/Table1`", "[[[999]]]", Scan); - CheckOrExec(helper, "SELECT count(ival) FROM `/Root/Table1`", "[[4u]]", Scan); - CheckOrExec(helper, "SELECT count(*) FROM `/Root/Table1`", "[[5u]]", Scan); - CheckOrExec(helper, "SELECT min(ival), max(ival), count(ival), count(*) FROM `/Root/Table1`", - "[[[-500];[999];4u;5u]]", Scan); + CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=1", "[[1;[10];[1000000]]]", Scan); + CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=2", "[[2;[20];[-500000]]]", Scan); + CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=3", "[[3;[30];#]]", Scan); + CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=4", "[[4;[40];[0]]]", Scan); + CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=5", TStringBuilder() << "[[5;[50];[" << MaxValidInterval << "]]]", Scan); + CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=6", TStringBuilder() << "[[6;[60];[" << MinValidInterval << "]]]", Scan); + CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY id", + TStringBuilder() << "[[1;[10];[1000000]];[2;[20];[-500000]];[3;[30];#];[4;[40];[0]];[5;[50];[" << MaxValidInterval + << "]];[6;[60];[" << MinValidInterval << "]]]", Scan); - { - const TString tableName2 = "/Root/Table2"; - TTestHelper::TColumnTable col2; - TVector<TTestHelper::TColumnSchema> schema2; - PrepareBase(helper, Table, tableName2, &col2, &schema2); - LoadData(helper, Table, Load, tableName2, - { { 1, 0, std::nullopt }, { 2, 0, std::nullopt } }, - &col2, &schema2); + CheckOrExec(helper, "SELECT ival FROM `/Root/Table1` WHERE id=1", "[[[1000000]]]", Scan); + CheckOrExec(helper, "SELECT ival FROM `/Root/Table1` WHERE id=3", "[[#]]", Scan); - CheckOrExec(helper, "SELECT min(ival), max(ival), count(ival), count(*) FROM `/Root/Table2`", - "[[#;#;0u;2u]]", Scan); - } + Base::LoadData(helper, Table, Load, tableName, { { 7, 0, std::nullopt } }, &col, &schema); + CheckOrExec(helper, "SELECT * FROM `/Root/Table1` WHERE id=7", "[[7;[0];#]]", Scan); + } + + Y_UNIT_TEST_SCENARIO(TestFilterEqual, FilterEqualScenario); + Y_UNIT_TEST_SCENARIO(TestFilterNulls, FilterNullsScenario); + Y_UNIT_TEST_SCENARIO(TestFilterCompare, FilterCompareScenario); + Y_UNIT_TEST_SCENARIO(TestOrderByInterval, OrderByScenario); + Y_UNIT_TEST_SCENARIO(TestGroupByInterval, GroupByScenario); + + Y_UNIT_TEST(TestAggregation, EQueryMode, ETableKind, ELoadKind) { + RunScenario<TIntervalTraits>(AggregationScenario(), Arg<0>(), Arg<1>(), Arg<2>()); + RunScenario<TIntervalTraits>(AggregationAllNullScenario(), Arg<0>(), Arg<1>(), Arg<2>()); } Y_UNIT_TEST(TestJoinById, EQueryMode, ETableKind, ELoadKind) { @@ -489,7 +337,7 @@ Y_UNIT_TEST_SUITE(KqpIntervalColumnShard) { const TString t1 = "/Root/Table1"; const TString t2 = "/Root/Table2"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); + TTestHelper helper(TIntervalTraits::CreateSettings()); TTestHelper::TColumnTable col1, col2; TVector<TTestHelper::TColumnSchema> s1, s2; if (Table == ETableKind::COLUMNSHARD) { @@ -511,7 +359,7 @@ Y_UNIT_TEST_SUITE(KqpIntervalColumnShard) { col2.SetName(t2).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s2); helper.CreateTable(col2); } else { - CreateDataShardTable(helper, t1); + Base::CreateDataShardTable(helper, t1); auto& session = helper.GetSession(); auto res = session .ExecuteSchemeQuery(TStringBuilder() << R"( @@ -526,7 +374,7 @@ Y_UNIT_TEST_SUITE(KqpIntervalColumnShard) { UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), NYdb::EStatus::SUCCESS); } - LoadData(helper, Table, Load, t1, + Base::LoadData(helper, Table, Load, t1, { { 1, 10, (i64)1000000 }, { 2, 20, (i64)-500000 }, { 3, 30, std::nullopt } }, &col1, &s1); @@ -606,122 +454,9 @@ Y_UNIT_TEST_SUITE(KqpIntervalColumnShard) { "[[1;[1000000];[2000000]];[1;[1000000];[3000000]];[2;[-500000];[-100000]]]", Scan); } - Y_UNIT_TEST(TestJoinByInterval, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - - const TString t1 = "/Root/JoinTable1"; - const TString t2 = "/Root/JoinTable2"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); - TTestHelper::TColumnTable col1, col2; - TVector<TTestHelper::TColumnSchema> s1, s2; - - if (Table == ETableKind::COLUMNSHARD) { - s1 = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("ival").SetType(NScheme::NTypeIds::Interval), - }; - - col1.SetName(t1).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s1); - helper.CreateTable(col1); - - s2 = s1; - col2.SetName(t2).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s2); - helper.CreateTable(col2); - } else { - CreateDataShardTable(helper, t1); - CreateDataShardTable(helper, t2); - } - - LoadData(helper, Table, Load, t1, - { { 1, 10, (i64)1000 }, { 2, 20, (i64)2000 }, { 3, 30, (i64)3000 } }, - &col1, &s1); - - LoadData(helper, Table, Load, t2, - { { 10, 100, (i64)1000 }, { 20, 200, (i64)2000 }, { 30, 300, (i64)4000 } }, - &col2, &s2); - - CheckOrExec(helper, - "SELECT t1.id, t2.id, t1.ival FROM `/Root/JoinTable1` AS t1 " - "JOIN `/Root/JoinTable2` AS t2 ON t1.ival = t2.ival " - "ORDER BY t1.id, t2.id", - "[[1;10;[1000]];[2;20;[2000]]]", Scan); - } - - Y_UNIT_TEST(TestOrderByWithLimit, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 10, (i64)500 }, { 2, 20, (i64)-200 }, { 3, 30, (i64)100 }, - { 4, 40, (i64)-300 }, { 5, 50, (i64)1000 }, { 6, 60, (i64)0 } }, - &col, &schema); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival LIMIT 1", - "[[4;[40];[-300]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival DESC LIMIT 1", - "[[5;[50];[1000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival LIMIT 3", - "[[4;[40];[-300]];[2;[20];[-200]];[6;[60];[0]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival DESC LIMIT 3", - "[[5;[50];[1000]];[1;[10];[500]];[3;[30];[100]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival LIMIT 2 OFFSET 2", - "[[6;[60];[0]];[3;[30];[100]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival LIMIT 100", - "[[4;[40];[-300]];[2;[20];[-200]];[6;[60];[0]];[3;[30];[100]];[1;[10];[500]];[5;[50];[1000]]]", Scan); - - CheckOrExec(helper, "SELECT * FROM `/Root/Table1` ORDER BY ival, id LIMIT 2", - "[[4;[40];[-300]];[2;[20];[-200]]]", Scan); - } - - Y_UNIT_TEST(TestGroupByWithNulls, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 10, (i64)100 }, { 2, 20, (i64)200 }, { 3, 30, std::nullopt }, - { 4, 40, (i64)100 }, { 5, 50, std::nullopt }, { 6, 60, (i64)200 }, - { 7, 70, std::nullopt }, { 8, 80, (i64)-100 } }, - &col, &schema); - - CheckOrExec(helper, "SELECT ival, count(*) AS cnt FROM `/Root/Table1` GROUP BY ival ORDER BY ival", - "[[#;3u];[[-100];1u];[[100];2u];[[200];2u]]", Scan); - - CheckOrExec(helper, "SELECT ival, count(ival) AS cnt FROM `/Root/Table1` GROUP BY ival ORDER BY ival", - "[[#;0u];[[-100];1u];[[100];2u];[[200];2u]]", Scan); - - CheckOrExec(helper, - "SELECT ival, count(*) AS cnt FROM `/Root/Table1` WHERE ival IS NOT NULL GROUP BY ival ORDER BY ival", - "[[[-100];1u];[[100];2u];[[200];2u]]", Scan); - - CheckOrExec(helper, "SELECT count(*) FROM `/Root/Table1` WHERE ival IS NULL", - "[[3u]]", Scan); - - CheckOrExec(helper, "SELECT min(ival), max(ival) FROM `/Root/Table1`", - "[[[-100];[200]]]", Scan); - } + Y_UNIT_TEST_JOIN_SCENARIO(TestJoinByInterval, JoinByIntervalScenario); + Y_UNIT_TEST_SCENARIO(TestOrderByWithLimit, OrderByWithLimitScenario); + Y_UNIT_TEST_SCENARIO(TestGroupByWithNulls, GroupByWithNullsScenario); Y_UNIT_TEST(TestIntervalAsPrimaryKey, EQueryMode, ETableKind, ELoadKind) { const auto Scan = Arg<0>(); @@ -729,7 +464,7 @@ Y_UNIT_TEST_SUITE(KqpIntervalColumnShard) { const auto Load = Arg<2>(); const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); + TTestHelper helper(TIntervalTraits::CreateSettings()); TTestHelper::TColumnTable col; TVector<TTestHelper::TColumnSchema> schema; PrepareIntervalPkTable(helper, Table, tableName, &col, &schema); @@ -764,14 +499,14 @@ Y_UNIT_TEST_SUITE(KqpIntervalColumnShard) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - auto runnerSettings = CreateKikimrSettingsWithIntervalSupport(); + auto runnerSettings = TIntervalTraits::CreateSettings(); runnerSettings.AppConfig.MutableTableServiceConfig()->SetEnableHtapTx(true); TTestHelper helper(runnerSettings); const TString ds = "/Root/RowSrc"; const TString cs = "/Root/ColSrc"; - CreateDataShardTable(helper, ds); + Base::CreateDataShardTable(helper, ds); TVector<TTestHelper::TColumnSchema> schema = { TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), @@ -856,10 +591,10 @@ Y_UNIT_TEST_SUITE(KqpIntervalColumnShard) { const auto Scan = Arg<0>(); const auto Table = Arg<1>(); - TTestHelper helper(CreateKikimrSettingsWithIntervalSupport()); + TTestHelper helper(TIntervalTraits::CreateSettings()); TTestHelper::TColumnTable col; TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, "/Root/Table1", &col, &schema); + Base::PrepareBase(helper, Table, "/Root/Table1", &col, &schema); { TStringBuilder builder; diff --git a/ydb/core/kqp/ut/olap/types/uuid_ut.cpp b/ydb/core/kqp/ut/olap/types/uuid_ut.cpp index ee14eac65e0..4aedb83e226 100644 --- a/ydb/core/kqp/ut/olap/types/uuid_ut.cpp +++ b/ydb/core/kqp/ut/olap/types/uuid_ut.cpp @@ -1,25 +1,8 @@ -#include "bool_test_enums.h" - -#include <ydb/core/formats/arrow/arrow_helpers.h> -#include <ydb/core/kqp/ut/common/columnshard.h> -#include <ydb/core/kqp/ut/common/kqp_ut_common.h> -#include <ydb/core/testlib/common_helper.h> -#include <ydb/core/testlib/cs_helper.h> -#include <ydb/core/tx/columnshard/hooks/testing/controller.h> -#include <ydb/core/tx/tx_proxy/proxy.h> - -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_replication.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/scheme/scheme.h> -#include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/client.h> +#include "column_type_scenarios.h" +#include "column_type_test_base.h" #include <yql/essentials/types/uuid/uuid.h> -#include <library/cpp/threading/local_executor/local_executor.h> -#include <util/generic/serialized_enum.h> -#include <util/string/printf.h> -#include <ydb/core/kqp/ut/common/arrow_builders.h> - namespace NKikimr { namespace NKqp { @@ -28,106 +11,6 @@ using namespace NYdb::NTable; Y_UNIT_TEST_SUITE(KqpUuidColumnShard) { namespace { - struct TRow { - i32 Id; - i64 IntVal; - std::optional<TString> Uid; - }; - - TKikimrSettings CreateKikimrSettingsWithUuidSupport() { - NKikimrConfig::TFeatureFlags featureFlags; - featureFlags.SetEnableColumnshardUuid(true); - return TKikimrSettings().SetWithSampleTables(false).SetFeatureFlags(featureFlags); - } - - void CreateDataShardTable(TTestHelper& helper, const TString& name) { - auto& session = helper.GetSession(); - auto res = session - .ExecuteSchemeQuery(TStringBuilder() << R"( - CREATE TABLE `)" << name << R"(` ( - id Int32 NOT NULL, - int Int64, - uid Uuid, - PRIMARY KEY (id) - ); - )") - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), NYdb::EStatus::SUCCESS); - } - - void CreateDataShardTableWithSecondColumn(TTestHelper& helper, const TString& name, const TString& secondName) { - auto& session = helper.GetSession(); - auto res = session - .ExecuteSchemeQuery(TStringBuilder() << R"( - CREATE TABLE `)" << name << R"(` ( - id Int32 NOT NULL, - )" << secondName << R"( Int64, - uid Uuid, - PRIMARY KEY (id) - ); - )") - .ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(res.GetStatus(), NYdb::EStatus::SUCCESS); - } - - void BulkUpsertRowTableYdbValueWithColumnName( - TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& columnName) { - TValueBuilder builder; - builder.BeginList(); - for (auto&& r : rows) { - builder.AddListItem().BeginStruct().AddMember("id").Int32(r.Id).AddMember(columnName).Int64(r.IntVal).AddMember("uid"); - if (r.Uid.has_value()) { - builder.BeginOptional().Uuid(TUuidValue(std::string(*r.Uid))).EndOptional(); - } else { - builder.EmptyOptional(EPrimitiveType::Uuid); - } - - builder.EndStruct(); - } - - builder.EndList(); - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, builder.Build()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void BulkUpsertRowTableYdbValue(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - BulkUpsertRowTableYdbValueWithColumnName(helper, name, rows, "int"); - } - - void BulkUpsertRowTableYdbValueWithSecondColumn( - TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& secondName) { - TValueBuilder builder; - builder.BeginList(); - for (auto&& r : rows) { - builder.AddListItem().BeginStruct().AddMember("id").Int32(r.Id).AddMember(secondName).Int64(r.IntVal).AddMember("uid"); - if (r.Uid.has_value()) { - builder.BeginOptional().Uuid(TUuidValue(std::string(*r.Uid))).EndOptional(); - } else { - builder.EmptyOptional(EPrimitiveType::Uuid); - } - - builder.EndStruct(); - } - - builder.EndList(); - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, builder.Build()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void BulkUpsertRowTableCSV(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - TStringBuilder builder; - for (auto&& r : rows) { - builder << r.Id << "," << r.IntVal << ","; - if (r.Uid.has_value()) { - builder << *r.Uid; - } - - builder << '\n'; - } - - auto result = helper.GetKikimr().GetTableClient().BulkUpsert(name, EDataFormat::CSV, builder).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } std::shared_ptr<arrow::Array> MakeUuidArrayNullable(const std::vector<std::optional<TString>>& uuidStrings) { auto type = arrow::fixed_size_binary(16); @@ -146,147 +29,46 @@ Y_UNIT_TEST_SUITE(KqpUuidColumnShard) { return out; } - std::shared_ptr<arrow::RecordBatch> MakeArrowBatchWithColumnName(const TVector<TRow>& rows, const TString& columnName) { - using namespace NKikimr::NKqp::NTestArrow; - std::vector<int32_t> ids; - std::vector<int64_t> vals; - std::vector<std::optional<TString>> uuids; - ids.reserve(rows.size()); - vals.reserve(rows.size()); - uuids.reserve(rows.size()); - for (auto&& r : rows) { - ids.push_back(r.Id); - vals.push_back(r.IntVal); - uuids.push_back(r.Uid); - } - - auto idArr = MakeInt32Array(ids); - auto intArr = MakeInt64Array(vals); - auto uuidArr = MakeUuidArrayNullable(uuids); - auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), /*nullable*/ false), - arrow::field(columnName, arrow::int64()), - arrow::field("uid", arrow::fixed_size_binary(16)) - }); - - return MakeBatch({ schema->field(0), schema->field(1), schema->field(2) }, { idArr, intArr, uuidArr }); - } - - std::shared_ptr<arrow::RecordBatch> MakeArrowBatch(const TVector<TRow>& rows) { - return MakeArrowBatchWithColumnName(rows, "int"); - } + struct TUuidTraits { + using TValue = TString; + static constexpr const char* ColumnName = "uid"; + static constexpr const char* SqlTypeName = "Uuid"; - std::shared_ptr<arrow::RecordBatch> MakeArrowBatchWithSecondColumn(const TVector<TRow>& rows, const TString& secondName) { - using namespace NKikimr::NKqp::NTestArrow; - std::vector<int32_t> ids; - std::vector<int64_t> seconds; - std::vector<std::optional<TString>> uuids; - ids.reserve(rows.size()); - seconds.reserve(rows.size()); - uuids.reserve(rows.size()); - for (auto&& r : rows) { - ids.push_back(r.Id); - seconds.push_back(r.IntVal); - uuids.push_back(r.Uid); + static TKikimrSettings CreateSettings() { + return CreateColumnshardSettings([](auto& f) { f.SetEnableColumnshardUuid(true); }); } - auto idArr = MakeInt32Array(ids); - auto secondArr = MakeInt64Array(seconds); - auto uuidArr = MakeUuidArrayNullable(uuids); - auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), /*nullable*/ false), - arrow::field(secondName, arrow::int64()), - arrow::field("uid", arrow::fixed_size_binary(16)) - }); - - return MakeBatch({ schema->field(0), schema->field(1), schema->field(2) }, { idArr, secondArr, uuidArr }); - } - - void BulkUpsertRowTableArrow(TTestHelper& helper, const TString& name, const TVector<TRow>& rows) { - auto batch = MakeArrowBatch(rows); - TString strBatch = NArrow::SerializeBatchNoCompression(batch); - TString strSchema = NArrow::SerializeSchema(*batch->schema()); - auto result = - helper.GetKikimr().GetTableClient().BulkUpsert(name, NYdb::NTable::EDataFormat::ApacheArrow, strBatch, strSchema).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void BulkUpsertRowTableArrowWithSecondColumn( - TTestHelper& helper, const TString& name, const TVector<TRow>& rows, const TString& secondName) { - auto batch = MakeArrowBatchWithSecondColumn(rows, secondName); - TString strBatch = NArrow::SerializeBatchNoCompression(batch); - TString strSchema = NArrow::SerializeSchema(*batch->schema()); - auto result = - helper.GetKikimr().GetTableClient().BulkUpsert(name, NYdb::NTable::EDataFormat::ApacheArrow, strBatch, strSchema).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - void LoadData(TTestHelper& helper, ETableKind table, ELoadKind load, const TString& name, const TVector<TRow>& rows, - TTestHelper::TColumnTable* col = nullptr, const TVector<TTestHelper::TColumnSchema>* schema = nullptr) { - switch (table) { - case ETableKind::COLUMNSHARD: { - Y_ABORT_UNLESS(col && schema); - if (load == ELoadKind::ARROW) { - TString columnName = "int"; - if (schema->size() >= 2) { - columnName = (*schema)[1].GetName(); - } + static auto GetTypeId() { return NScheme::NTypeIds::Uuid; } - auto batch = MakeArrowBatchWithColumnName(rows, columnName); - helper.BulkUpsert(*col, batch); - } else if (load == ELoadKind::YDB_VALUE) { - TString columnName = "int"; - if (schema->size() >= 2) { - columnName = (*schema)[1].GetName(); - } - - BulkUpsertRowTableYdbValueWithColumnName(helper, name, rows, columnName); - } else { - BulkUpsertRowTableCSV(helper, name, rows); - } - - break; + static void AppendYdbValue(TValueBuilder& builder, const std::optional<TString>& val) { + if (val.has_value()) { + builder.BeginOptional().Uuid(TUuidValue(std::string(*val))).EndOptional(); + } else { + builder.EmptyOptional(EPrimitiveType::Uuid); } - case ETableKind::DATASHARD: { - if (load == ELoadKind::ARROW) { - BulkUpsertRowTableArrow(helper, name, rows); - } else if (load == ELoadKind::YDB_VALUE) { - BulkUpsertRowTableYdbValue(helper, name, rows); - } else { - BulkUpsertRowTableCSV(helper, name, rows); - } + } - break; + static void AppendCsvValue(TStringBuilder& builder, const std::optional<TString>& val) { + if (val.has_value()) { + builder << *val; } } - } - void CheckOrExec(TTestHelper& helper, const TString& query, const TString& expected, EQueryMode scanMode) { - if (scanMode == EQueryMode::SCAN_QUERY) { - helper.ReadData(query, expected); - } else { - helper.ReadDataExecQuery(query, expected); + static std::shared_ptr<arrow::Array> MakeArrowArray(const TVector<TTypedRow<TString>>& rows) { + std::vector<std::optional<TString>> uuids; + uuids.reserve(rows.size()); + for (auto&& r : rows) { + uuids.push_back(r.TypedVal); + } + return MakeUuidArrayNullable(uuids); } - } - void PrepareBase(TTestHelper& helper, ETableKind tableKind, const TString& tableName, TTestHelper::TColumnTable* colTableOut, - TVector<TTestHelper::TColumnSchema>* schemaOut) { - if (tableKind == ETableKind::COLUMNSHARD) { - TVector<TTestHelper::TColumnSchema> schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("uid").SetType(NScheme::NTypeIds::Uuid), - }; + static std::shared_ptr<arrow::DataType> ArrowType() { return arrow::fixed_size_binary(16); } - *schemaOut = schema; - TTestHelper::TColumnTable col; - col.SetName(tableName).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema); - helper.CreateTable(col); - *colTableOut = col; - } else { - CreateDataShardTable(helper, tableName); - } - } + static void LoadPkTable(TTestHelper& helper, ELoadKind load, const TString& tableName, TTestHelper::TColumnTable& table); + }; + + COLUMN_TYPE_TEST_USING(TUuidTraits); static const TString UUID1 = "550e8400-e29b-41d4-a716-446655440000"; static const TString UUID2 = "550e8400-e29b-41d4-a716-446655440001"; @@ -300,173 +82,210 @@ Y_UNIT_TEST_SUITE(KqpUuidColumnShard) { static const TString UUID_D = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"; static const TString UUID_E = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"; -} // namespace - - Y_UNIT_TEST(TestSimpleQueries, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 10, UUID1 }, { 2, 20, UUID4 }, { 3, 30, std::nullopt } }, &col, &schema); - - CheckOrExec(helper, "SELECT id, int FROM `/Root/Table1` WHERE id=1", "[[1;[10]]]", Scan); - CheckOrExec(helper, "SELECT id, int FROM `/Root/Table1` WHERE id=3", "[[3;[30]]]", Scan); - CheckOrExec(helper, "SELECT id, int FROM `/Root/Table1` ORDER BY id", - "[[1;[10]];[2;[20]];[3;[30]]]", Scan); - CheckOrExec(helper, - "SELECT uid, id FROM `/Root/Table1` WHERE id=1", - R"([[["550e8400-e29b-41d4-a716-446655440000"];1]])", Scan); - CheckOrExec(helper, - "SELECT uid, id FROM `/Root/Table1` WHERE id=3", - R"([[#;3]])", Scan); + void TUuidTraits::LoadPkTable(TTestHelper& helper, ELoadKind load, const TString& tableName, TTestHelper::TColumnTable& table) { + if (load == ELoadKind::ARROW) { + std::vector<std::optional<TString>> uuids = { UUID_A, UUID_B, UUID_C }; + auto uuidArr = MakeUuidArrayNullable(uuids); + using namespace NKikimr::NKqp::NTestArrow; + auto valArr = MakeInt64Array({ (int64_t)1, (int64_t)2, (int64_t)3 }); + auto batch = MakeBatch( + { arrow::field("uid", arrow::fixed_size_binary(16), /*nullable*/ false), + arrow::field("val", arrow::int64()) }, + { uuidArr, valArr } + ); + helper.BulkUpsert(table, batch); + } else if (load == ELoadKind::YDB_VALUE) { + TValueBuilder builder; + builder.BeginList(); + builder.AddListItem().BeginStruct().AddMember("uid").Uuid(TUuidValue(std::string(UUID_A))).AddMember("val").Int64(1).EndStruct(); + builder.AddListItem().BeginStruct().AddMember("uid").Uuid(TUuidValue(std::string(UUID_B))).AddMember("val").Int64(2).EndStruct(); + builder.AddListItem().BeginStruct().AddMember("uid").Uuid(TUuidValue(std::string(UUID_C))).AddMember("val").Int64(3).EndStruct(); + builder.EndList(); + auto res = helper.GetKikimr().GetTableClient().BulkUpsert(tableName, builder.Build()).GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); + } else { + TStringBuilder csv; + csv << "11111111-1111-1111-1111-111111111111,1\n"; + csv << "22222222-2222-2222-2222-222222222222,2\n"; + csv << "33333333-3333-3333-3333-333333333333,3\n"; + auto res = helper.GetKikimr().GetTableClient().BulkUpsert(tableName, EDataFormat::CSV, csv).GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); + } } - Y_UNIT_TEST(TestFilterEqual, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 0, UUID1 }, { 2, 0, UUID4 }, { 3, 0, UUID5 } }, &col, &schema); - - CheckOrExec(helper, - R"(SELECT id FROM `/Root/Table1` WHERE uid = CAST("550e8400-e29b-41d4-a716-446655440000" AS Uuid))", - "[[1]]", Scan); - CheckOrExec(helper, - R"(SELECT id FROM `/Root/Table1` WHERE uid != CAST("550e8400-e29b-41d4-a716-446655440000" AS Uuid) ORDER BY id)", - "[[2];[3]]", Scan); - CheckOrExec(helper, - R"(SELECT id FROM `/Root/Table1` WHERE uid = CAST("660e8400-e29b-41d4-a716-446655440000" AS Uuid))", - "[[2]]", Scan); + TScenario<TString> FilterEqualScenario() { + return { + "/Root/Table1", + { { 1, 0, UUID1 }, { 2, 0, UUID4 }, { 3, 0, UUID5 } }, + { + { R"(SELECT id FROM `/Root/Table1` WHERE uid = CAST("550e8400-e29b-41d4-a716-446655440000" AS Uuid))", "[[1]]" }, + { R"(SELECT id FROM `/Root/Table1` WHERE uid != CAST("550e8400-e29b-41d4-a716-446655440000" AS Uuid) ORDER BY id)", "[[2];[3]]" }, + { R"(SELECT id FROM `/Root/Table1` WHERE uid = CAST("660e8400-e29b-41d4-a716-446655440000" AS Uuid))", "[[2]]" }, + }, + }; } - Y_UNIT_TEST(TestFilterNulls, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 0, UUID1 }, { 2, 0, std::nullopt }, { 3, 0, UUID5 }, { 4, 0, std::nullopt } }, &col, &schema); - - CheckOrExec(helper, "SELECT id FROM `/Root/Table1` WHERE uid IS NULL ORDER BY id", - "[[2];[4]]", Scan); - CheckOrExec(helper, "SELECT id FROM `/Root/Table1` WHERE uid IS NOT NULL ORDER BY id", - "[[1];[3]]", Scan); + TScenario<TString> FilterNullsScenario() { + return { + "/Root/Table1", + { { 1, 0, UUID1 }, { 2, 0, std::nullopt }, { 3, 0, UUID5 }, { 4, 0, std::nullopt } }, + { + { "SELECT id FROM `/Root/Table1` WHERE uid IS NULL ORDER BY id", "[[2];[4]]" }, + { "SELECT id FROM `/Root/Table1` WHERE uid IS NOT NULL ORDER BY id", "[[1];[3]]" }, + }, + }; } - Y_UNIT_TEST(TestFilterCompare, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 0, UUID_A }, { 2, 0, UUID_B }, { 3, 0, UUID_C }, { 4, 0, UUID_D } }, &col, &schema); - - CheckOrExec(helper, - R"(SELECT id FROM `/Root/Table1` WHERE uid < CAST("33333333-3333-3333-3333-333333333333" AS Uuid) ORDER BY id)", - "[[1];[2]]", Scan); - - CheckOrExec(helper, - R"(SELECT id FROM `/Root/Table1` WHERE uid > CAST("22222222-2222-2222-2222-222222222222" AS Uuid) ORDER BY id)", - "[[3];[4]]", Scan); + TScenario<TString> FilterCompareScenario() { + return { + "/Root/Table1", + { { 1, 0, UUID_A }, { 2, 0, UUID_B }, { 3, 0, UUID_C }, { 4, 0, UUID_D } }, + { + { R"(SELECT id FROM `/Root/Table1` WHERE uid < CAST("33333333-3333-3333-3333-333333333333" AS Uuid) ORDER BY id)", "[[1];[2]]" }, + { R"(SELECT id FROM `/Root/Table1` WHERE uid > CAST("22222222-2222-2222-2222-222222222222" AS Uuid) ORDER BY id)", "[[3];[4]]" }, + { R"(SELECT id FROM `/Root/Table1` WHERE uid <= CAST("22222222-2222-2222-2222-222222222222" AS Uuid) ORDER BY id)", "[[1];[2]]" }, + { R"(SELECT id FROM `/Root/Table1` WHERE uid >= CAST("33333333-3333-3333-3333-333333333333" AS Uuid) ORDER BY id)", "[[3];[4]]" }, + }, + }; + } - CheckOrExec(helper, - R"(SELECT id FROM `/Root/Table1` WHERE uid <= CAST("22222222-2222-2222-2222-222222222222" AS Uuid) ORDER BY id)", - "[[1];[2]]", Scan); + TScenario<TString> OrderByScenario() { + return { + "/Root/Table1", + { { 1, 10, UUID_C }, { 2, 20, UUID_A }, { 3, 30, UUID_B } }, + { + { "SELECT uid, id FROM `/Root/Table1` ORDER BY uid", + R"([[["11111111-1111-1111-1111-111111111111"];2];[["22222222-2222-2222-2222-222222222222"];3];[["33333333-3333-3333-3333-333333333333"];1]])" }, + { "SELECT uid, id FROM `/Root/Table1` ORDER BY uid DESC", + R"([[["33333333-3333-3333-3333-333333333333"];1];[["22222222-2222-2222-2222-222222222222"];3];[["11111111-1111-1111-1111-111111111111"];2]])" }, + }, + }; + } - CheckOrExec(helper, - R"(SELECT id FROM `/Root/Table1` WHERE uid >= CAST("33333333-3333-3333-3333-333333333333" AS Uuid) ORDER BY id)", - "[[3];[4]]", Scan); + TScenario<TString> GroupByScenario() { + return { + "/Root/Table1", + { { 1, 0, UUID_D }, { 2, 0, UUID_E }, { 3, 0, UUID_D }, { 4, 0, UUID_E }, { 5, 0, UUID1 } }, + { + { "SELECT uid, count(*) AS cnt FROM `/Root/Table1` GROUP BY uid ORDER BY uid", + R"([[["550e8400-e29b-41d4-a716-446655440000"];1u];[["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"];2u];[["bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"];2u]])" }, + }, + }; } - Y_UNIT_TEST(TestOrderBy, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); + TScenario<TString> AggregationScenario() { + return { + "/Root/Table1", + { { 1, 0, UUID_A }, { 2, 0, UUID_C }, { 3, 0, UUID_B }, { 4, 0, std::nullopt } }, + { + { "SELECT min(uid) FROM `/Root/Table1`", R"([[["11111111-1111-1111-1111-111111111111"]]])" }, + { "SELECT max(uid) FROM `/Root/Table1`", R"([[["33333333-3333-3333-3333-333333333333"]]])" }, + { "SELECT count(uid) FROM `/Root/Table1`", "[[3u]]" }, + { "SELECT count(*) FROM `/Root/Table1`", "[[4u]]" }, + }, + }; + } - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 10, UUID_C }, { 2, 20, UUID_A }, { 3, 30, UUID_B } }, &col, &schema); + TJoinScenario<TString> JoinByUuidScenario() { + return { + "/Root/Table1", + "/Root/Table2", + { { 1, 10, UUID_D }, { 2, 20, UUID_E } }, + { { 1, 100, UUID_E }, { 2, 200, UUID_D }, { 3, 300, UUID_A } }, + { + { R"(SELECT t1.id, t2.id, t1.uid FROM `/Root/Table1` AS t1 JOIN `/Root/Table2` AS t2 ON t1.uid = t2.uid ORDER BY t1.id, t2.id)", + R"([[1;2;["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]];[2;1;["bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"]]])" }, + }, + }; + } - CheckOrExec(helper, - "SELECT uid, id FROM `/Root/Table1` ORDER BY uid", - R"([[["11111111-1111-1111-1111-111111111111"];2];[["22222222-2222-2222-2222-222222222222"];3];[["33333333-3333-3333-3333-333333333333"];1]])", Scan); + TScenario<TString> OrderByWithLimitScenario() { + return { + "/Root/Table1", + { { 1, 0, UUID_C }, { 2, 0, UUID_A }, { 3, 0, UUID_B }, { 4, 0, UUID_D } }, + { + { "SELECT uid, id FROM `/Root/Table1` ORDER BY uid LIMIT 2", + R"([[["11111111-1111-1111-1111-111111111111"];2];[["22222222-2222-2222-2222-222222222222"];3]])" }, + { "SELECT uid, id FROM `/Root/Table1` ORDER BY uid DESC LIMIT 1", + R"([[["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"];4]])" }, + { "SELECT uid, id FROM `/Root/Table1` ORDER BY uid LIMIT 3", + R"([[["11111111-1111-1111-1111-111111111111"];2];[["22222222-2222-2222-2222-222222222222"];3];[["33333333-3333-3333-3333-333333333333"];1]])" }, + }, + }; + } - CheckOrExec(helper, - "SELECT uid, id FROM `/Root/Table1` ORDER BY uid DESC", - R"([[["33333333-3333-3333-3333-333333333333"];1];[["22222222-2222-2222-2222-222222222222"];3];[["11111111-1111-1111-1111-111111111111"];2]])", Scan); + TScenario<TString> GroupByWithNullsScenario() { + return { + "/Root/Table1", + { { 1, 0, UUID_D }, { 2, 0, std::nullopt }, { 3, 0, UUID_D }, { 4, 0, std::nullopt }, { 5, 0, UUID_E } }, + { + { "SELECT uid, count(*) AS cnt FROM `/Root/Table1` GROUP BY uid ORDER BY uid", + R"([[#;2u];[["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"];2u];[["bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"];1u]])" }, + }, + }; } - Y_UNIT_TEST(TestGroupBy, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); + TPkLookupScenario PkLookupScenario() { + const TString tableName = "/Root/ColumnTableTest"; + return { + tableName, + { + { R"(SELECT val FROM `/Root/ColumnTableTest` WHERE uid = CAST("22222222-2222-2222-2222-222222222222" AS Uuid))", "[[[2]]]" }, + { R"(SELECT val FROM `/Root/ColumnTableTest` WHERE uid = CAST("11111111-1111-1111-1111-111111111111" AS Uuid))", "[[[1]]]" }, + { R"(SELECT val FROM `/Root/ColumnTableTest` WHERE uid = CAST("33333333-3333-3333-3333-333333333333" AS Uuid))", "[[[3]]]" }, + }, + }; + } + TCsvScenario CsvScenario() { const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 0, UUID_D }, { 2, 0, UUID_E }, { 3, 0, UUID_D }, { 4, 0, UUID_E }, { 5, 0, UUID1 } }, &col, &schema); - - CheckOrExec(helper, - "SELECT uid, count(*) AS cnt FROM `/Root/Table1` GROUP BY uid ORDER BY uid", - R"([[["550e8400-e29b-41d4-a716-446655440000"];1u];[["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"];2u];[["bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"];2u]])", Scan); + return { + tableName, + "1,0,550e8400-e29b-41d4-a716-446655440000\n" + "2,0,660e8400-e29b-41d4-a716-446655440000\n" + "3,0,770e8400-e29b-41d4-a716-446655440000\n", + { + { "SELECT id FROM `" + tableName + "` ORDER BY id", "[[1];[2];[3]]" }, + { "SELECT uid, id FROM `" + tableName + "` ORDER BY id", + R"([[["550e8400-e29b-41d4-a716-446655440000"];1];[["660e8400-e29b-41d4-a716-446655440000"];2];[["770e8400-e29b-41d4-a716-446655440000"];3]])" }, + }, + }; } - Y_UNIT_TEST(TestAggregation, EQueryMode, ETableKind, ELoadKind) { +} // namespace + + Y_UNIT_TEST(TestSimpleQueries, EQueryMode, ETableKind, ELoadKind) { const auto Scan = Arg<0>(); const auto Table = Arg<1>(); const auto Load = Arg<2>(); const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); + TTestHelper helper(TUuidTraits::CreateSettings()); TTestHelper::TColumnTable col; TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 0, UUID_A }, { 2, 0, UUID_C }, { 3, 0, UUID_B }, { 4, 0, std::nullopt } }, &col, &schema); + Base::PrepareBase(helper, Table, tableName, &col, &schema); + Base::LoadData(helper, Table, Load, tableName, + { { 1, 10, UUID1 }, { 2, 20, UUID4 }, { 3, 30, std::nullopt } }, &col, &schema); + CheckOrExec(helper, "SELECT id, int FROM `/Root/Table1` WHERE id=1", "[[1;[10]]]", Scan); + CheckOrExec(helper, "SELECT id, int FROM `/Root/Table1` WHERE id=3", "[[3;[30]]]", Scan); + CheckOrExec(helper, "SELECT id, int FROM `/Root/Table1` ORDER BY id", + "[[1;[10]];[2;[20]];[3;[30]]]", Scan); CheckOrExec(helper, - "SELECT min(uid) FROM `/Root/Table1`", - R"([[["11111111-1111-1111-1111-111111111111"]]])", Scan); - CheckOrExec(helper, - "SELECT max(uid) FROM `/Root/Table1`", - R"([[["33333333-3333-3333-3333-333333333333"]]])", Scan); - CheckOrExec(helper, - "SELECT count(uid) FROM `/Root/Table1`", - "[[3u]]", Scan); + "SELECT uid, id FROM `/Root/Table1` WHERE id=1", + R"([[["550e8400-e29b-41d4-a716-446655440000"];1]])", Scan); CheckOrExec(helper, - "SELECT count(*) FROM `/Root/Table1`", - "[[4u]]", Scan); + "SELECT uid, id FROM `/Root/Table1` WHERE id=3", + R"([[#;3]])", Scan); } + Y_UNIT_TEST_SCENARIO(TestFilterEqual, FilterEqualScenario); + Y_UNIT_TEST_SCENARIO(TestFilterNulls, FilterNullsScenario); + Y_UNIT_TEST_SCENARIO(TestFilterCompare, FilterCompareScenario); + Y_UNIT_TEST_SCENARIO(TestOrderBy, OrderByScenario); + Y_UNIT_TEST_SCENARIO(TestGroupBy, GroupByScenario); + Y_UNIT_TEST_SCENARIO(TestAggregation, AggregationScenario); + Y_UNIT_TEST(TestJoinById, EQueryMode, ETableKind, ELoadKind) { const auto Scan = Arg<0>(); const auto Table = Arg<1>(); @@ -474,7 +293,7 @@ Y_UNIT_TEST_SUITE(KqpUuidColumnShard) { const TString t1 = "/Root/Table1"; const TString t2 = "/Root/Table2"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); + TTestHelper helper(TUuidTraits::CreateSettings()); TTestHelper::TColumnTable col1, col2; TVector<TTestHelper::TColumnSchema> s1, s2; if (Table == ETableKind::COLUMNSHARD) { @@ -495,19 +314,19 @@ Y_UNIT_TEST_SUITE(KqpUuidColumnShard) { col2.SetName(t2).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s2); helper.CreateTable(col2); } else { - CreateDataShardTable(helper, t1); - CreateDataShardTableWithSecondColumn(helper, t2, "table1_id"); + Base::CreateDataShardTable(helper, t1); + Base::CreateDataShardTableWithSecondColumn(helper, t2, "table1_id"); } - LoadData(helper, Table, Load, t1, { { 1, 100, UUID1 }, { 2, 200, UUID4 } }, &col1, &s1); + Base::LoadData(helper, Table, Load, t1, { { 1, 100, UUID1 }, { 2, 200, UUID4 } }, &col1, &s1); if (Table == ETableKind::COLUMNSHARD) { - LoadData(helper, Table, Load, t2, { { 1, 1, UUID_D }, { 2, 1, UUID_E }, { 3, 2, UUID_A } }, &col2, &s2); + Base::LoadData(helper, Table, Load, t2, { { 1, 1, UUID_D }, { 2, 1, UUID_E }, { 3, 2, UUID_A } }, &col2, &s2); } else { if (Load == ELoadKind::ARROW) { - BulkUpsertRowTableArrowWithSecondColumn( + Base::BulkUpsertRowTableArrowWithSecondColumn( helper, t2, { { 1, 1, UUID_D }, { 2, 1, UUID_E }, { 3, 2, UUID_A } }, "table1_id"); } else if (Load == ELoadKind::YDB_VALUE) { - BulkUpsertRowTableYdbValueWithSecondColumn( + Base::BulkUpsertRowTableYdbValueWithSecondColumn( helper, t2, { { 1, 1, UUID_D }, { 2, 1, UUID_E }, { 3, 2, UUID_A } }, "table1_id"); } else { TStringBuilder csv; @@ -524,150 +343,16 @@ Y_UNIT_TEST_SUITE(KqpUuidColumnShard) { R"([[1;["550e8400-e29b-41d4-a716-446655440000"];["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]];[1;["550e8400-e29b-41d4-a716-446655440000"];["bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"]];[2;["660e8400-e29b-41d4-a716-446655440000"];["11111111-1111-1111-1111-111111111111"]]])", Scan); } - Y_UNIT_TEST(TestJoinByUuid, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString t1 = "/Root/Table1"; - const TString t2 = "/Root/Table2"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col1, col2; - TVector<TTestHelper::TColumnSchema> s1, s2; - if (Table == ETableKind::COLUMNSHARD) { - s1 = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("uid").SetType(NScheme::NTypeIds::Uuid), - }; - - col1.SetName(t1).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s1); - helper.CreateTable(col1); - s2 = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("int").SetType(NScheme::NTypeIds::Int64), - TTestHelper::TColumnSchema().SetName("uid").SetType(NScheme::NTypeIds::Uuid), - }; - - col2.SetName(t2).SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(s2); - helper.CreateTable(col2); - } else { - CreateDataShardTable(helper, t1); - CreateDataShardTable(helper, t2); - } - - LoadData(helper, Table, Load, t1, { { 1, 10, UUID_D }, { 2, 20, UUID_E } }, &col1, &s1); - LoadData(helper, Table, Load, t2, { { 1, 100, UUID_E }, { 2, 200, UUID_D }, { 3, 300, UUID_A } }, &col2, &s2); - - CheckOrExec(helper, - R"(SELECT t1.id, t2.id, t1.uid FROM `/Root/Table1` AS t1 JOIN `/Root/Table2` AS t2 ON t1.uid = t2.uid ORDER BY t1.id, t2.id)", - R"([[1;2;["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]];[2;1;["bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"]]])", Scan); - } - - Y_UNIT_TEST(TestOrderByWithLimit, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 0, UUID_C }, { 2, 0, UUID_A }, { 3, 0, UUID_B }, { 4, 0, UUID_D } }, &col, &schema); - - CheckOrExec(helper, - "SELECT uid, id FROM `/Root/Table1` ORDER BY uid LIMIT 2", - R"([[["11111111-1111-1111-1111-111111111111"];2];[["22222222-2222-2222-2222-222222222222"];3]])", Scan); - - CheckOrExec(helper, - "SELECT uid, id FROM `/Root/Table1` ORDER BY uid DESC LIMIT 1", - R"([[["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"];4]])", Scan); - - CheckOrExec(helper, - "SELECT uid, id FROM `/Root/Table1` ORDER BY uid LIMIT 3", - R"([[["11111111-1111-1111-1111-111111111111"];2];[["22222222-2222-2222-2222-222222222222"];3];[["33333333-3333-3333-3333-333333333333"];1]])", Scan); - } - - Y_UNIT_TEST(TestGroupByWithNulls, EQueryMode, ETableKind, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - const auto Load = Arg<2>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - LoadData(helper, Table, Load, tableName, - { { 1, 0, UUID_D }, { 2, 0, std::nullopt }, { 3, 0, UUID_D }, { 4, 0, std::nullopt }, { 5, 0, UUID_E } }, &col, &schema); - - CheckOrExec(helper, - "SELECT uid, count(*) AS cnt FROM `/Root/Table1` GROUP BY uid ORDER BY uid", - R"([[#;2u];[["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"];2u];[["bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"];1u]])", Scan); - } - - Y_UNIT_TEST(TestUuidAsPrimaryKey, EQueryMode, ELoadKind) { - const auto Scan = Arg<0>(); - const auto Load = Arg<1>(); - - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - - TVector<TTestHelper::TColumnSchema> schema = { - TTestHelper::TColumnSchema().SetName("uid").SetType(NScheme::NTypeIds::Uuid).SetNullable(false), - TTestHelper::TColumnSchema().SetName("val").SetType(NScheme::NTypeIds::Int64) - }; - - TTestHelper::TColumnTable testTable; - testTable.SetName("/Root/ColumnTableTest").SetPrimaryKey({"uid"}).SetSharding({"uid"}).SetSchema(schema); - helper.CreateTable(testTable); - - if (Load == ELoadKind::ARROW) { - std::vector<std::optional<TString>> uuids = { UUID_A, UUID_B, UUID_C }; - auto uuidArr = MakeUuidArrayNullable(uuids); - using namespace NKikimr::NKqp::NTestArrow; - auto valArr = MakeInt64Array({ (int64_t)1, (int64_t)2, (int64_t)3 }); - auto batch = MakeBatch( - { arrow::field("uid", arrow::fixed_size_binary(16), /*nullable*/ false), - arrow::field("val", arrow::int64()) }, - { uuidArr, valArr } - ); - helper.BulkUpsert(testTable, batch); - } else if (Load == ELoadKind::YDB_VALUE) { - TValueBuilder builder; - builder.BeginList(); - builder.AddListItem().BeginStruct().AddMember("uid").Uuid(TUuidValue(std::string(UUID_A))).AddMember("val").Int64(1).EndStruct(); - builder.AddListItem().BeginStruct().AddMember("uid").Uuid(TUuidValue(std::string(UUID_B))).AddMember("val").Int64(2).EndStruct(); - builder.AddListItem().BeginStruct().AddMember("uid").Uuid(TUuidValue(std::string(UUID_C))).AddMember("val").Int64(3).EndStruct(); - builder.EndList(); - auto res = helper.GetKikimr().GetTableClient().BulkUpsert("/Root/ColumnTableTest", builder.Build()).GetValueSync(); - UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); - } else { - TStringBuilder csv; - csv << "11111111-1111-1111-1111-111111111111,1" << Endl; - csv << "22222222-2222-2222-2222-222222222222,2" << Endl; - csv << "33333333-3333-3333-3333-333333333333,3" << Endl; - auto res = helper.GetKikimr().GetTableClient().BulkUpsert("/Root/ColumnTableTest", EDataFormat::CSV, csv).GetValueSync(); - UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); - } - - CheckOrExec(helper, - R"(SELECT val FROM `/Root/ColumnTableTest` WHERE uid = CAST("22222222-2222-2222-2222-222222222222" AS Uuid))", - "[[[2]]]", Scan); - CheckOrExec(helper, - R"(SELECT val FROM `/Root/ColumnTableTest` WHERE uid = CAST("11111111-1111-1111-1111-111111111111" AS Uuid))", - "[[[1]]]", Scan); - CheckOrExec(helper, - R"(SELECT val FROM `/Root/ColumnTableTest` WHERE uid = CAST("33333333-3333-3333-3333-333333333333" AS Uuid))", - "[[[3]]]", Scan); - } + Y_UNIT_TEST_JOIN_SCENARIO(TestJoinByUuid, JoinByUuidScenario); + Y_UNIT_TEST_SCENARIO(TestOrderByWithLimit, OrderByWithLimitScenario); + Y_UNIT_TEST_SCENARIO(TestGroupByWithNulls, GroupByWithNullsScenario); + Y_UNIT_TEST_PK_SCENARIO(TestUuidAsPrimaryKey, PkLookupScenario); Y_UNIT_TEST(TestDmlParityAndCTAS, EQueryMode, ELoadKind) { const auto Scan = Arg<0>(); const auto Load = Arg<1>(); - auto runnerSettings = CreateKikimrSettingsWithUuidSupport(); + auto runnerSettings = TUuidTraits::CreateSettings(); runnerSettings.AppConfig.MutableTableServiceConfig()->SetEnableHtapTx(true); TTestHelper helper(runnerSettings); @@ -753,31 +438,7 @@ Y_UNIT_TEST_SUITE(KqpUuidColumnShard) { R"([[["550e8400-e29b-41d4-a716-446655440000"];1;[10]];[["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"];2;[20]];[["bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"];4;[40]]])", Scan); } - Y_UNIT_TEST(TestCsv, EQueryMode, ETableKind) { - const auto Scan = Arg<0>(); - const auto Table = Arg<1>(); - - const TString tableName = "/Root/Table1"; - TTestHelper helper(CreateKikimrSettingsWithUuidSupport()); - TTestHelper::TColumnTable col; - TVector<TTestHelper::TColumnSchema> schema; - PrepareBase(helper, Table, tableName, &col, &schema); - - { - TStringBuilder builder; - builder << "1,0,550e8400-e29b-41d4-a716-446655440000" << Endl; - builder << "2,0,660e8400-e29b-41d4-a716-446655440000" << Endl; - builder << "3,0,770e8400-e29b-41d4-a716-446655440000" << Endl; - const auto result = helper.GetKikimr().GetTableClient().BulkUpsert( - tableName, EDataFormat::CSV, builder).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - CheckOrExec(helper, "SELECT id FROM `" + tableName + "` ORDER BY id", "[[1];[2];[3]]", Scan); - CheckOrExec(helper, - "SELECT uid, id FROM `" + tableName + "` ORDER BY id", - R"([[["550e8400-e29b-41d4-a716-446655440000"];1];[["660e8400-e29b-41d4-a716-446655440000"];2];[["770e8400-e29b-41d4-a716-446655440000"];3]])", Scan); - } + Y_UNIT_TEST_CSV_SCENARIO(TestCsv, CsvScenario); } } // namespace NKqp diff --git a/ydb/core/kqp/ut/olap/types/ya.make b/ydb/core/kqp/ut/olap/types/ya.make index 1be271a945d..6f7d9bf1253 100644 --- a/ydb/core/kqp/ut/olap/types/ya.make +++ b/ydb/core/kqp/ut/olap/types/ya.make @@ -39,6 +39,6 @@ PEERDIR( YQL_LAST_ABI_VERSION() -GENERATE_ENUM_SERIALIZATION(bool_test_enums.h) +GENERATE_ENUM_SERIALIZATION(column_type_test_enums.h) END() |
