diff options
| author | monster <monster@ydb.tech> | 2022-10-04 14:16:50 +0300 |
|---|---|---|
| committer | monster <monster@ydb.tech> | 2022-10-04 14:16:50 +0300 |
| commit | 4780349153062fe2a64aa5bd878b927bdc532d94 (patch) | |
| tree | f949a8e639271c4035eff6244e47ba757960e3d1 | |
| parent | cd0a32136bce815fff11b9787ab5b058c2cce7d6 (diff) | |
| download | ydb-4780349153062fe2a64aa5bd878b927bdc532d94.tar.gz | |
initial pg types support in local db
573 files changed, 3597 insertions, 1807 deletions
diff --git a/ydb/core/actorlib_impl/ut/CMakeLists.darwin.txt b/ydb/core/actorlib_impl/ut/CMakeLists.darwin.txt index f720b5a034a..221666064f8 100644 --- a/ydb/core/actorlib_impl/ut/CMakeLists.darwin.txt +++ b/ydb/core/actorlib_impl/ut/CMakeLists.darwin.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-actorlib_impl-ut PUBLIC library-cpp-getopt library-cpp-svnversion core-testlib-actors - core-testlib-basics + testlib-basics-default ) target_link_options(ydb-core-actorlib_impl-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/actorlib_impl/ut/CMakeLists.linux.txt b/ydb/core/actorlib_impl/ut/CMakeLists.linux.txt index ad984648c01..f6c32c2dc18 100644 --- a/ydb/core/actorlib_impl/ut/CMakeLists.linux.txt +++ b/ydb/core/actorlib_impl/ut/CMakeLists.linux.txt @@ -27,7 +27,7 @@ target_link_libraries(ydb-core-actorlib_impl-ut PUBLIC library-cpp-getopt library-cpp-svnversion core-testlib-actors - core-testlib-basics + testlib-basics-default ) target_link_options(ydb-core-actorlib_impl-ut PRIVATE -ldl diff --git a/ydb/core/blobstorage/dsproxy/ut/CMakeLists.darwin.txt b/ydb/core/blobstorage/dsproxy/ut/CMakeLists.darwin.txt index 505bfc907f6..d25ea7f0bab 100644 --- a/ydb/core/blobstorage/dsproxy/ut/CMakeLists.darwin.txt +++ b/ydb/core/blobstorage/dsproxy/ut/CMakeLists.darwin.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-blobstorage-dsproxy-ut PUBLIC blobstorage-ut_vdisk-lib blobstorage-vdisk-common blobstorage-vdisk-query - ydb-core-testlib + core-testlib-default core-testlib-actors core-testlib-basics ) diff --git a/ydb/core/blobstorage/dsproxy/ut/CMakeLists.linux.txt b/ydb/core/blobstorage/dsproxy/ut/CMakeLists.linux.txt index bdff919b557..7a0176498c3 100644 --- a/ydb/core/blobstorage/dsproxy/ut/CMakeLists.linux.txt +++ b/ydb/core/blobstorage/dsproxy/ut/CMakeLists.linux.txt @@ -28,7 +28,7 @@ target_link_libraries(ydb-core-blobstorage-dsproxy-ut PUBLIC blobstorage-ut_vdisk-lib blobstorage-vdisk-common blobstorage-vdisk-query - ydb-core-testlib + core-testlib-default core-testlib-actors core-testlib-basics ) diff --git a/ydb/core/blobstorage/dsproxy/ut_fat/CMakeLists.darwin.txt b/ydb/core/blobstorage/dsproxy/ut_fat/CMakeLists.darwin.txt index f470ddf13e5..2edad9dd725 100644 --- a/ydb/core/blobstorage/dsproxy/ut_fat/CMakeLists.darwin.txt +++ b/ydb/core/blobstorage/dsproxy/ut_fat/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-blobstorage-dsproxy-ut_fat PUBLIC core-blobstorage-pdisk core-blobstorage-vdisk blobstorage-vdisk-common - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-blobstorage-dsproxy-ut_fat PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/blobstorage/dsproxy/ut_fat/CMakeLists.linux.txt b/ydb/core/blobstorage/dsproxy/ut_fat/CMakeLists.linux.txt index 8419abbec4e..1eb24584a9b 100644 --- a/ydb/core/blobstorage/dsproxy/ut_fat/CMakeLists.linux.txt +++ b/ydb/core/blobstorage/dsproxy/ut_fat/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-blobstorage-dsproxy-ut_fat PUBLIC core-blobstorage-pdisk core-blobstorage-vdisk blobstorage-vdisk-common - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-blobstorage-dsproxy-ut_fat PRIVATE -ldl diff --git a/ydb/core/blobstorage/nodewarden/ut/CMakeLists.darwin.txt b/ydb/core/blobstorage/nodewarden/ut/CMakeLists.darwin.txt index 0fb64510bd5..8d0ae85f1ab 100644 --- a/ydb/core/blobstorage/nodewarden/ut/CMakeLists.darwin.txt +++ b/ydb/core/blobstorage/nodewarden/ut/CMakeLists.darwin.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-core-blobstorage-nodewarden-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main core-blobstorage-nodewarden - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-blobstorage-nodewarden-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/blobstorage/nodewarden/ut/CMakeLists.linux.txt b/ydb/core/blobstorage/nodewarden/ut/CMakeLists.linux.txt index 9c40dd66fd6..5d8e0d0f682 100644 --- a/ydb/core/blobstorage/nodewarden/ut/CMakeLists.linux.txt +++ b/ydb/core/blobstorage/nodewarden/ut/CMakeLists.linux.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-blobstorage-nodewarden-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main core-blobstorage-nodewarden - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-blobstorage-nodewarden-ut PRIVATE -ldl diff --git a/ydb/core/blobstorage/nodewarden/ut_sequence/CMakeLists.darwin.txt b/ydb/core/blobstorage/nodewarden/ut_sequence/CMakeLists.darwin.txt index 1ba91724ddf..56195156049 100644 --- a/ydb/core/blobstorage/nodewarden/ut_sequence/CMakeLists.darwin.txt +++ b/ydb/core/blobstorage/nodewarden/ut_sequence/CMakeLists.darwin.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-core-blobstorage-nodewarden-ut_sequence PUBLIC core-blobstorage-crypto core-blobstorage-nodewarden core-blobstorage-pdisk - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-blobstorage-nodewarden-ut_sequence PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/blobstorage/nodewarden/ut_sequence/CMakeLists.linux.txt b/ydb/core/blobstorage/nodewarden/ut_sequence/CMakeLists.linux.txt index ddd10efe7fd..5bbf8dd0058 100644 --- a/ydb/core/blobstorage/nodewarden/ut_sequence/CMakeLists.linux.txt +++ b/ydb/core/blobstorage/nodewarden/ut_sequence/CMakeLists.linux.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-blobstorage-nodewarden-ut_sequence PUBLIC core-blobstorage-crypto core-blobstorage-nodewarden core-blobstorage-pdisk - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-blobstorage-nodewarden-ut_sequence PRIVATE -ldl diff --git a/ydb/core/blobstorage/storagepoolmon/ut/CMakeLists.darwin.txt b/ydb/core/blobstorage/storagepoolmon/ut/CMakeLists.darwin.txt index dbf6c826c16..7fb27c04e72 100644 --- a/ydb/core/blobstorage/storagepoolmon/ut/CMakeLists.darwin.txt +++ b/ydb/core/blobstorage/storagepoolmon/ut/CMakeLists.darwin.txt @@ -16,7 +16,7 @@ target_link_libraries(ydb-core-blobstorage-storagepoolmon-ut PUBLIC library-cpp-getopt library-cpp-svnversion core-blobstorage-storagepoolmon - ydb-core-testlib + core-testlib-default core-testlib-actors core-testlib-basics ) diff --git a/ydb/core/blobstorage/storagepoolmon/ut/CMakeLists.linux.txt b/ydb/core/blobstorage/storagepoolmon/ut/CMakeLists.linux.txt index e88dac66b06..44937a0cbf5 100644 --- a/ydb/core/blobstorage/storagepoolmon/ut/CMakeLists.linux.txt +++ b/ydb/core/blobstorage/storagepoolmon/ut/CMakeLists.linux.txt @@ -18,7 +18,7 @@ target_link_libraries(ydb-core-blobstorage-storagepoolmon-ut PUBLIC library-cpp-getopt library-cpp-svnversion core-blobstorage-storagepoolmon - ydb-core-testlib + core-testlib-default core-testlib-actors core-testlib-basics ) diff --git a/ydb/core/blobstorage/ut_vdisk/CMakeLists.darwin.txt b/ydb/core/blobstorage/ut_vdisk/CMakeLists.darwin.txt index 3c6079fa054..2c7e9d55a85 100644 --- a/ydb/core/blobstorage/ut_vdisk/CMakeLists.darwin.txt +++ b/ydb/core/blobstorage/ut_vdisk/CMakeLists.darwin.txt @@ -29,6 +29,7 @@ target_link_libraries(ydb-core-blobstorage-ut_vdisk PUBLIC ydb-core-erasure ydb-core-scheme udf-service-stub + yql-sql-pg_dummy ) target_link_options(ydb-core-blobstorage-ut_vdisk PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/blobstorage/ut_vdisk/CMakeLists.linux.txt b/ydb/core/blobstorage/ut_vdisk/CMakeLists.linux.txt index 0086d887385..71a49269d92 100644 --- a/ydb/core/blobstorage/ut_vdisk/CMakeLists.linux.txt +++ b/ydb/core/blobstorage/ut_vdisk/CMakeLists.linux.txt @@ -31,6 +31,7 @@ target_link_libraries(ydb-core-blobstorage-ut_vdisk PUBLIC ydb-core-erasure ydb-core-scheme udf-service-stub + yql-sql-pg_dummy ) target_link_options(ydb-core-blobstorage-ut_vdisk PRIVATE -ldl diff --git a/ydb/core/blobstorage/vdisk/skeleton/ut/CMakeLists.darwin.txt b/ydb/core/blobstorage/vdisk/skeleton/ut/CMakeLists.darwin.txt index 318f73ce482..7a99a0da1f8 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/ut/CMakeLists.darwin.txt +++ b/ydb/core/blobstorage/vdisk/skeleton/ut/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-blobstorage-vdisk-skeleton-ut PUBLIC library-cpp-svnversion ydb-core-blobstorage blobstorage-vdisk-common - ydb-core-testlib + core-testlib-default core-testlib-actors ) target_link_options(ydb-core-blobstorage-vdisk-skeleton-ut PRIVATE diff --git a/ydb/core/blobstorage/vdisk/skeleton/ut/CMakeLists.linux.txt b/ydb/core/blobstorage/vdisk/skeleton/ut/CMakeLists.linux.txt index bd07535b2d4..aa0faa9574d 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/ut/CMakeLists.linux.txt +++ b/ydb/core/blobstorage/vdisk/skeleton/ut/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-blobstorage-vdisk-skeleton-ut PUBLIC library-cpp-svnversion ydb-core-blobstorage blobstorage-vdisk-common - ydb-core-testlib + core-testlib-default core-testlib-actors ) target_link_options(ydb-core-blobstorage-vdisk-skeleton-ut PRIVATE diff --git a/ydb/core/client/minikql_compile/db_key_resolver.h b/ydb/core/client/minikql_compile/db_key_resolver.h index 4d93c1a4c25..8304aa1cb58 100644 --- a/ydb/core/client/minikql_compile/db_key_resolver.h +++ b/ydb/core/client/minikql_compile/db_key_resolver.h @@ -42,7 +42,7 @@ public: struct TColumn { ui32 Column; i32 KeyPosition; - ui32 Type; + NKikimr::NScheme::TTypeInfo Type; ui32 AllowInplaceMode; NKikimr::EColumnTypeConstraint TypeConstraint; }; diff --git a/ydb/core/client/minikql_compile/ut/CMakeLists.darwin.txt b/ydb/core/client/minikql_compile/ut/CMakeLists.darwin.txt index 8179879db9d..c8d3034db0b 100644 --- a/ydb/core/client/minikql_compile/ut/CMakeLists.darwin.txt +++ b/ydb/core/client/minikql_compile/ut/CMakeLists.darwin.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-client-minikql_compile-ut PUBLIC core-client-scheme_cache_lib core-client-server ydb-core-tablet - ydb-core-testlib + core-testlib-default library-yql-minikql udf-service-exception_policy ) diff --git a/ydb/core/client/minikql_compile/ut/CMakeLists.linux.txt b/ydb/core/client/minikql_compile/ut/CMakeLists.linux.txt index e7535e44a61..f1e362978ef 100644 --- a/ydb/core/client/minikql_compile/ut/CMakeLists.linux.txt +++ b/ydb/core/client/minikql_compile/ut/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-client-minikql_compile-ut PUBLIC core-client-scheme_cache_lib core-client-server ydb-core-tablet - ydb-core-testlib + core-testlib-default library-yql-minikql udf-service-exception_policy ) diff --git a/ydb/core/client/minikql_compile/yql_expr_minikql.cpp b/ydb/core/client/minikql_compile/yql_expr_minikql.cpp index 28a7a4dbff9..c025b8fe73e 100644 --- a/ydb/core/client/minikql_compile/yql_expr_minikql.cpp +++ b/ydb/core/client/minikql_compile/yql_expr_minikql.cpp @@ -392,15 +392,19 @@ private: YQL_ENSURE(column); typeConstraint = column->TypeConstraint; + // TODO: support pg types + auto columnTypeId = column->Type.GetTypeId(); + YQL_ENSURE(columnTypeId != NScheme::NTypeIds::Pg, "pg types are not supported"); + // Decimal type is transformed into parametrized Decimal(22, 9). - if (column->Type == NYql::NProto::TypeIds::Decimal) { + if (columnTypeId == NYql::NProto::TypeIds::Decimal) { columnDataType = ctx.MakeType<TDataExprParamsType>( - NUdf::GetDataSlot(column->Type), + NUdf::GetDataSlot(columnTypeId), ToString(NScheme::DECIMAL_PRECISION), ToString(NScheme::DECIMAL_SCALE)); } else { columnDataType = GetMkqlDataTypeAnnotation( - TDataType::Create(column->Type, *MkqlCtx->TypeEnv), + TDataType::Create(columnTypeId, *MkqlCtx->TypeEnv), ctx); } } @@ -839,7 +843,8 @@ void FillColumnsToRead(IDbSchemeResolver::TTableResult* lookup, TExprNode* selec auto columnName = selectTuple->Child(i)->Content(); const auto& systemColumn = GetSystemColumns().find(columnName); if (systemColumn != GetSystemColumns().end()) { - columnsToRead.emplace_back(columnName, systemColumn->second.ColumnId, systemColumn->second.TypeId, EColumnTypeConstraint::Nullable); + columnsToRead.emplace_back(columnName, systemColumn->second.ColumnId, + NScheme::TTypeInfo(systemColumn->second.TypeId), EColumnTypeConstraint::Nullable); } else { auto column = lookup->Columns.FindPtr(columnName); YQL_ENSURE(column); @@ -921,7 +926,7 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr auto rowTuple = node.Child(1); TVector<TRuntimeNode> row(rowTuple->ChildrenSize()); - TVector<ui32> keyTypes(rowTuple->ChildrenSize()); + TVector<NScheme::TTypeInfo> keyTypes(rowTuple->ChildrenSize()); for (ui32 i = 0; i < rowTuple->ChildrenSize(); ++i) { auto columnName = rowTuple->Child(i)->Child(0)->Content(); auto column = lookup->Columns.FindPtr(columnName); @@ -992,7 +997,7 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr } TTableRangeOptions options = mkqlContext->PgmBuilder->GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(Max(fromComponents, toComponents)); + TVector<NScheme::TTypeInfo> keyTypes(Max(fromComponents, toComponents)); TVector<TRuntimeNode> from(fromComponents); TVector<TRuntimeNode> to(toComponents); ui32 keyIndex = 0; @@ -1080,7 +1085,7 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr auto rowTuple = node.Child(1); TVector<TRuntimeNode> row(rowTuple->ChildrenSize()); - TVector<ui32> keyTypes(rowTuple->ChildrenSize()); + TVector<NScheme::TTypeInfo> keyTypes(rowTuple->ChildrenSize()); for (ui32 i = 0; i < rowTuple->ChildrenSize(); ++i) { auto columnName = rowTuple->Child(i)->Child(0)->Content(); auto column = lookup->Columns.FindPtr(columnName); @@ -1131,7 +1136,7 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr auto rowTuple = node.Child(1); TVector<TRuntimeNode> row(rowTuple->ChildrenSize()); - TVector<ui32> keyTypes(rowTuple->ChildrenSize()); + TVector<NScheme::TTypeInfo> keyTypes(rowTuple->ChildrenSize()); for (ui32 i = 0; i < rowTuple->ChildrenSize(); ++i) { auto columnName = rowTuple->Child(i)->Child(0)->Content(); auto column = lookup->Columns.FindPtr(columnName); diff --git a/ydb/core/client/minikql_compile/yql_expr_minikql_compile_ut.cpp b/ydb/core/client/minikql_compile/yql_expr_minikql_compile_ut.cpp index df56ab1c273..052ca5f6f96 100644 --- a/ydb/core/client/minikql_compile/yql_expr_minikql_compile_ut.cpp +++ b/ydb/core/client/minikql_compile/yql_expr_minikql_compile_ut.cpp @@ -11,6 +11,8 @@ namespace NYql { using namespace NKikimr; using namespace NKikimr::NMiniKQL; +using TTypeInfo = NScheme::TTypeInfo; + namespace { const TDuration TIME_LIMIT = TDuration::Seconds(60); @@ -72,8 +74,8 @@ namespace { table.Table.ColumnNames = { "key", "value" }; table.TableId.Reset(new TTableId(1, 2)); table.KeyColumnCount = 1; - table.Columns.insert(std::make_pair("key", TColumn{ 34, 0, NUdf::TDataType<ui32>::Id, 0, EColumnTypeConstraint::Nullable })); - table.Columns.insert(std::make_pair("value", TColumn{ 56, -1, NUdf::TDataType<char*>::Id, (ui32)EInplaceUpdateMode::Min, EColumnTypeConstraint::Nullable })); + table.Columns.insert(std::make_pair("key", TColumn{ 34, 0, TTypeInfo(NUdf::TDataType<ui32>::Id), 0, EColumnTypeConstraint::Nullable })); + table.Columns.insert(std::make_pair("value", TColumn{ 56, -1, TTypeInfo(NUdf::TDataType<char*>::Id), (ui32)EInplaceUpdateMode::Min, EColumnTypeConstraint::Nullable })); services.DbSchemeResolver.AddTable(table); IDbSchemeResolver::TTableResult table2(IDbSchemeResolver::TTableResult::Ok); @@ -81,8 +83,8 @@ namespace { table2.Table.ColumnNames = { "key", "value" }; table2.TableId.Reset(new TTableId(10, 20)); table2.KeyColumnCount = 1; - table2.Columns.insert(std::make_pair("key", TColumn{ 340, 0, NUdf::TDataType<ui32>::Id, 0, EColumnTypeConstraint::Nullable })); - table2.Columns.insert(std::make_pair("value", TColumn{ 560, -1, NUdf::TDataType<char*>::Id, (ui32)EInplaceUpdateMode::Min, EColumnTypeConstraint::Nullable })); + table2.Columns.insert(std::make_pair("key", TColumn{ 340, 0, TTypeInfo(NUdf::TDataType<ui32>::Id), 0, EColumnTypeConstraint::Nullable })); + table2.Columns.insert(std::make_pair("value", TColumn{ 560, -1, TTypeInfo(NUdf::TDataType<char*>::Id), (ui32)EInplaceUpdateMode::Min, EColumnTypeConstraint::Nullable })); services.DbSchemeResolver.AddTable(table2); } } @@ -107,7 +109,7 @@ Y_UNIT_TEST_SUITE(TTestYqlToMiniKQLCompile) { UNIT_ASSERT_VALUES_EQUAL(res[0].Columns["value"].Column, 56); UNIT_ASSERT_VALUES_EQUAL(res[0].Columns["value"].KeyPosition, -1); UNIT_ASSERT_VALUES_EQUAL(res[0].Columns["value"].AllowInplaceMode, (ui32)EInplaceUpdateMode::Min); - UNIT_ASSERT_VALUES_EQUAL(res[0].Columns["value"].Type, (ui32)NUdf::TDataType<char*>::Id); + UNIT_ASSERT_VALUES_EQUAL(res[0].Columns["value"].Type.GetTypeId(), (ui32)NUdf::TDataType<char*>::Id); } Y_UNIT_TEST(OnlyResult) { diff --git a/ydb/core/client/minikql_result_lib/ut/CMakeLists.darwin.txt b/ydb/core/client/minikql_result_lib/ut/CMakeLists.darwin.txt index d1521ad2cb2..fc6819f6ce0 100644 --- a/ydb/core/client/minikql_result_lib/ut/CMakeLists.darwin.txt +++ b/ydb/core/client/minikql_result_lib/ut/CMakeLists.darwin.txt @@ -21,7 +21,7 @@ target_link_libraries(ydb-core-client-minikql_result_lib-ut PUBLIC cpp-testing-unittest_main core-client-minikql_result_lib cpp-testing-unittest - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-client-minikql_result_lib-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/client/minikql_result_lib/ut/CMakeLists.linux.txt b/ydb/core/client/minikql_result_lib/ut/CMakeLists.linux.txt index 6de99596903..230f018b3bc 100644 --- a/ydb/core/client/minikql_result_lib/ut/CMakeLists.linux.txt +++ b/ydb/core/client/minikql_result_lib/ut/CMakeLists.linux.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-client-minikql_result_lib-ut PUBLIC cpp-testing-unittest_main core-client-minikql_result_lib cpp-testing-unittest - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-client-minikql_result_lib-ut PRIVATE -ldl diff --git a/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.cpp b/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.cpp index d019533c69e..7d0dfdbf50c 100644 --- a/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.cpp +++ b/ydb/core/client/scheme_cache_lib/yql_db_scheme_resolver.cpp @@ -67,7 +67,7 @@ class TTableProxyActor : public TActorBootstrapped<TTableProxyActor> { if (systemColumn != GetSystemColumns().end()) { reply.Columns.insert({ column, - {systemColumn->second.ColumnId, -1, systemColumn->second.TypeId, 0, EColumnTypeConstraint::Nullable} + {systemColumn->second.ColumnId, -1, NScheme::TTypeInfo(systemColumn->second.TypeId), 0, EColumnTypeConstraint::Nullable} }); continue; } diff --git a/ydb/core/client/server/CMakeLists.txt b/ydb/core/client/server/CMakeLists.txt index 502f7f22e4d..3c8a48ea36f 100644 --- a/ydb/core/client/server/CMakeLists.txt +++ b/ydb/core/client/server/CMakeLists.txt @@ -32,10 +32,12 @@ target_link_libraries(core-client-server PUBLIC core-engine-minikql ydb-core-grpc_services ydb-core-keyvalue + core-kqp-common ydb-core-node_whiteboard ydb-core-persqueue core-persqueue-writer ydb-core-protos + ydb-core-scheme ydb-core-ydb_convert core-ymq-actor ydb-library-aclib diff --git a/ydb/core/client/server/msgbus_server_db.cpp b/ydb/core/client/server/msgbus_server_db.cpp index ff851a58975..f3f5d032050 100644 --- a/ydb/core/client/server/msgbus_server_db.cpp +++ b/ydb/core/client/server/msgbus_server_db.cpp @@ -384,7 +384,7 @@ public: const TVector<const NTxProxy::TTableColumnInfo*>& keys, const THashMap<TString, const NTxProxy::TTableColumnInfo*>& columnByName, THashSet<TString> notNullColumns, TVector<NMiniKQL::TRuntimeNode>& result) { TVector<NMiniKQL::TRuntimeNode> keyColumns; - TVector<ui32> keyTypes; + TVector<NScheme::TTypeInfo> keyTypes; keyTypes.reserve(keys.size()); for (const NTxProxy::TTableColumnInfo* key : keys) { Y_VERIFY(key != nullptr); @@ -395,8 +395,10 @@ public: keyColumns.reserve(keys.size()); for (const NTxProxy::TTableColumnInfo* key : keys) { NJson::TJsonValue jsonKey; + // TODO: support pg types + Y_VERIFY(key->PType.GetTypeId() != NScheme::NTypeIds::Pg, "pg types are not supported"); if (jsonWhere.GetValue(key->Name, &jsonKey)) { - keyColumns.emplace_back(NewDataLiteral(pgmBuilder, jsonKey, key->PType)); + keyColumns.emplace_back(NewDataLiteral(pgmBuilder, jsonKey, key->PType.GetTypeId())); } else { throw yexception() << "Key \"" << key->Name << "\" was not specified"; } @@ -451,7 +453,9 @@ public: NMiniKQL::TTableRangeOptions tableRangeOptions = pgmBuilder.GetDefaultTableRangeOptions(); TVector<NMiniKQL::TRuntimeNode> keyFromColumns = keyColumns; for (size_t i = keyColumns.size(); i < keyTypes.size(); ++i) { - keyFromColumns.emplace_back(pgmBuilder.NewEmptyOptionalDataLiteral(keyTypes[i])); + // TODO: support pg types + Y_VERIFY(keyTypes[i].GetTypeId() != NScheme::NTypeIds::Pg, "pg types are not supported"); + keyFromColumns.emplace_back(pgmBuilder.NewEmptyOptionalDataLiteral(keyTypes[i].GetTypeId())); } tableRangeOptions.ToColumns = keyColumns; tableRangeOptions.FromColumns = keyFromColumns; @@ -467,7 +471,9 @@ public: auto itCol = columnByName.find(itVal->first); if (itCol != columnByName.end()) { auto update = pgmBuilder.GetUpdateRowBuilder(); - update.SetColumn(itCol->second->Id, itCol->second->PType, pgmBuilder.NewOptional(NewDataLiteral(pgmBuilder, itVal->second, itCol->second->PType))); + // TODO: support pg types + Y_VERIFY(itCol->second->PType.GetTypeId() != NScheme::NTypeIds::Pg, "pg types are not supported"); + update.SetColumn(itCol->second->Id, itCol->second->PType, pgmBuilder.NewOptional(NewDataLiteral(pgmBuilder, itVal->second, itCol->second->PType.GetTypeId()))); pgmReturn = pgmBuilder.Append(pgmReturn, pgmBuilder.UpdateRow(tableInfo.TableId, keyTypes, keyColumns, update)); } else { throw yexception() << "Column \"" << itVal->first << "\" not found"; diff --git a/ydb/core/client/server/msgbus_server_s3_listing.cpp b/ydb/core/client/server/msgbus_server_s3_listing.cpp index 94542d8d737..0e58a57e1c3 100644 --- a/ydb/core/client/server/msgbus_server_s3_listing.cpp +++ b/ydb/core/client/server/msgbus_server_s3_listing.cpp @@ -8,6 +8,8 @@ #include <ydb/core/actorlib_impl/long_timer.h> #include <ydb/core/grpc_services/rpc_calls.h> #include <ydb/core/ydb_convert/ydb_convert.h> +#include <ydb/core/kqp/common/kqp_types.h> +#include <ydb/core/scheme/scheme_type_info.h> #include <util/system/unaligned_mem.h> namespace NKikimr { @@ -34,7 +36,7 @@ private: bool WaitingResolveReply; bool Finished; TAutoPtr<NSchemeCache::TSchemeCacheNavigate> ResolveNamesResult; - TVector<NScheme::TTypeId> KeyColumnTypes; + TVector<NScheme::TTypeInfo> KeyColumnTypes; TSysTables::TTableColumnInfo PathColumnInfo; TVector<TSysTables::TTableColumnInfo> CommonPrefixesColumns; TVector<TSysTables::TTableColumnInfo> ContentsColumns; @@ -194,7 +196,7 @@ private: TString errStr; TVector<TCell> prefixCells; - TConstArrayRef<NScheme::TTypeId> prefixTypes(KeyColumnTypes.data(), KeyColumnTypes.size() - 1); // -1 for path column + TConstArrayRef<NScheme::TTypeInfo> prefixTypes(KeyColumnTypes.data(), KeyColumnTypes.size() - 1); // -1 for path column NMiniKQL::CellsFromTuple(&Request->GetKeyPrefix().GetType(), Request->GetKeyPrefix().GetValue(), prefixTypes, true, prefixCells, errStr); if (!errStr.empty()) { @@ -208,7 +210,7 @@ private: ui32 pathColPos = prefixCells.size(); Y_VERIFY(pathColPos < KeyColumnTypes.size()); PathColumnInfo = entry.Columns[keyColumnIds[pathColPos]]; - if (PathColumnInfo.PType != NScheme::NTypeIds::Utf8) { + if (PathColumnInfo.PType.GetTypeId() != NScheme::NTypeIds::Utf8) { ReplyWithError(MSTATUS_ERROR, NTxProxy::TResultStatus::EStatus::WrongRequest, Sprintf("Value for path column '%s' has type %s, expected Utf8", PathColumnInfo.Name.data(), NScheme::TypeName(PathColumnInfo.PType)), ctx); @@ -218,7 +220,7 @@ private: CommonPrefixesColumns.push_back(PathColumnInfo); TVector<TCell> suffixCells; - TConstArrayRef<NScheme::TTypeId> suffixTypes(KeyColumnTypes.data() + pathColPos, KeyColumnTypes.size() - pathColPos); // starts at path column + TConstArrayRef<NScheme::TTypeInfo> suffixTypes(KeyColumnTypes.data() + pathColPos, KeyColumnTypes.size() - pathColPos); // starts at path column NMiniKQL::CellsFromTuple(&Request->GetStartAfterKeySuffix().GetType(), Request->GetStartAfterKeySuffix().GetValue(), suffixTypes, true, suffixCells, errStr); if (!errStr.empty()) { @@ -489,8 +491,7 @@ private: col->SetName(colInfo.Name); col->MutableType()->SetKind(NKikimrMiniKQL::Optional); auto* item = col->MutableType()->MutableOptional()->MutableItem(); - item->SetKind(NKikimrMiniKQL::Data); - item->MutableData()->SetScheme(colInfo.PType); + NScheme::ProtoMiniKQLTypeFromTypeInfo(item, colInfo.PType); } void BuildResultType(NKikimrMiniKQL::TType& type) const { diff --git a/ydb/core/client/server/ut/CMakeLists.darwin.txt b/ydb/core/client/server/ut/CMakeLists.darwin.txt index 094ca6798cd..62b99bcc13e 100644 --- a/ydb/core/client/server/ut/CMakeLists.darwin.txt +++ b/ydb/core/client/server/ut/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-client-server-ut PUBLIC cpp-testing-gmock_in_unittest ydb-core-persqueue ydb-core-tablet_flat - ydb-core-testlib + core-testlib-default core-testlib-actors ) target_link_options(ydb-core-client-server-ut PRIVATE diff --git a/ydb/core/client/server/ut/CMakeLists.linux.txt b/ydb/core/client/server/ut/CMakeLists.linux.txt index d87ac09ac67..d51915d33ae 100644 --- a/ydb/core/client/server/ut/CMakeLists.linux.txt +++ b/ydb/core/client/server/ut/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-client-server-ut PUBLIC cpp-testing-gmock_in_unittest ydb-core-persqueue ydb-core-tablet_flat - ydb-core-testlib + core-testlib-default core-testlib-actors ) target_link_options(ydb-core-client-server-ut PRIVATE diff --git a/ydb/core/client/ut/CMakeLists.darwin.txt b/ydb/core/client/ut/CMakeLists.darwin.txt index adca63db791..e749d361178 100644 --- a/ydb/core/client/ut/CMakeLists.darwin.txt +++ b/ydb/core/client/ut/CMakeLists.darwin.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-client-ut PUBLIC library-cpp-svnversion core-client-scheme_cache_lib test-libs-rows - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-client-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/client/ut/CMakeLists.linux.txt b/ydb/core/client/ut/CMakeLists.linux.txt index a174a026163..95a38a926f1 100644 --- a/ydb/core/client/ut/CMakeLists.linux.txt +++ b/ydb/core/client/ut/CMakeLists.linux.txt @@ -27,7 +27,7 @@ target_link_libraries(ydb-core-client-ut PUBLIC library-cpp-svnversion core-client-scheme_cache_lib test-libs-rows - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-client-ut PRIVATE -ldl diff --git a/ydb/core/cms/console/ut/CMakeLists.darwin.txt b/ydb/core/cms/console/ut/CMakeLists.darwin.txt index 8daffabf380..68b10f5a0f8 100644 --- a/ydb/core/cms/console/ut/CMakeLists.darwin.txt +++ b/ydb/core/cms/console/ut/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-cms-console-ut PUBLIC cpp-protobuf-util cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-cms-console-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/cms/console/ut/CMakeLists.linux.txt b/ydb/core/cms/console/ut/CMakeLists.linux.txt index b30e2dbb6a5..b07660d0841 100644 --- a/ydb/core/cms/console/ut/CMakeLists.linux.txt +++ b/ydb/core/cms/console/ut/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-cms-console-ut PUBLIC cpp-protobuf-util cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-cms-console-ut PRIVATE -ldl diff --git a/ydb/core/cms/ut/CMakeLists.darwin.txt b/ydb/core/cms/ut/CMakeLists.darwin.txt index c35c8bed546..acf38cecda7 100644 --- a/ydb/core/cms/ut/CMakeLists.darwin.txt +++ b/ydb/core/cms/ut/CMakeLists.darwin.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-cms-ut PUBLIC ydb-core-cms library-cpp-getopt library-cpp-svnversion - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-cms-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/cms/ut/CMakeLists.linux.txt b/ydb/core/cms/ut/CMakeLists.linux.txt index f07fb4aad4f..a325a16f3fc 100644 --- a/ydb/core/cms/ut/CMakeLists.linux.txt +++ b/ydb/core/cms/ut/CMakeLists.linux.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-cms-ut PUBLIC ydb-core-cms library-cpp-getopt library-cpp-svnversion - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-cms-ut PRIVATE -ldl diff --git a/ydb/core/cms/ut_sentinel/CMakeLists.darwin.txt b/ydb/core/cms/ut_sentinel/CMakeLists.darwin.txt index 26e890c5981..fdc9505a7f2 100644 --- a/ydb/core/cms/ut_sentinel/CMakeLists.darwin.txt +++ b/ydb/core/cms/ut_sentinel/CMakeLists.darwin.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-cms-ut_sentinel PUBLIC ydb-core-cms library-cpp-getopt library-cpp-svnversion - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-cms-ut_sentinel PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/cms/ut_sentinel/CMakeLists.linux.txt b/ydb/core/cms/ut_sentinel/CMakeLists.linux.txt index 10142ee779c..e99f6ab04ed 100644 --- a/ydb/core/cms/ut_sentinel/CMakeLists.linux.txt +++ b/ydb/core/cms/ut_sentinel/CMakeLists.linux.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-cms-ut_sentinel PUBLIC ydb-core-cms library-cpp-getopt library-cpp-svnversion - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-cms-ut_sentinel PRIVATE -ldl diff --git a/ydb/core/engine/kikimr_program_builder.cpp b/ydb/core/engine/kikimr_program_builder.cpp index e4077e2e102..6b50565fd8d 100644 --- a/ydb/core/engine/kikimr_program_builder.cpp +++ b/ydb/core/engine/kikimr_program_builder.cpp @@ -23,15 +23,17 @@ TType* ValidateColumns( rowTypeBuilder.Reserve(columns.size()); tagsBuilder.Reserve(columns.size()); for (auto& col : columns) { - MKQL_ENSURE(col.SchemeType != 0, "Null type is not allowed"); + // TODO: support pg types + MKQL_ENSURE(col.SchemeType.GetTypeId() != 0, "Null type is not allowed"); + MKQL_ENSURE(col.SchemeType.GetTypeId() != NScheme::NTypeIds::Pg, "pg types are not supported"); TType* dataType; - if (col.SchemeType == NYql::NProto::TypeIds::Decimal) + if (col.SchemeType.GetTypeId() == NYql::NProto::TypeIds::Decimal) dataType = TDataDecimalType::Create( NScheme::DECIMAL_PRECISION, NScheme::DECIMAL_SCALE, builder->GetTypeEnvironment()); else - dataType = TDataType::Create(col.SchemeType, builder->GetTypeEnvironment()); + dataType = TDataType::Create(col.SchemeType.GetTypeId(), builder->GetTypeEnvironment()); if (col.TypeConstraint == EColumnTypeConstraint::Nullable) { dataType = TOptionalType::Create(dataType, builder->GetTypeEnvironment()); @@ -103,24 +105,30 @@ TUpdateRowBuilder::TUpdateRowBuilder(const TTypeEnvironment& env) } void TUpdateRowBuilder::SetColumn( - ui32 columnId, ui32 expectedType, + ui32 columnId, NScheme::TTypeInfo expectedType, TRuntimeNode value) { + // TODO: support pg types + MKQL_ENSURE(expectedType.GetTypeId() != NScheme::NTypeIds::Pg, "pg types are not supported"); + bool isOptional; - value = DoRewriteNullType(value, expectedType, NullInternName, Env); + value = DoRewriteNullType(value, expectedType.GetTypeId(), NullInternName, Env); TDataType* dataType = UnpackOptionalData(value, isOptional); - MKQL_ENSURE(expectedType == dataType->GetSchemeType(), - "Mismatch of column type expectedType = " << expectedType + + MKQL_ENSURE(expectedType.GetTypeId() == dataType->GetSchemeType(), + "Mismatch of column type expectedType = " << expectedType.GetTypeId() << " actual type = " << dataType->GetSchemeType()); Builder.Add(ToString(columnId), value); } void TUpdateRowBuilder::InplaceUpdateColumn( - ui32 columnId, ui32 expectedType, + ui32 columnId, NScheme::TTypeInfo expectedType, TRuntimeNode value, EInplaceUpdateMode mode) { - MKQL_ENSURE(AS_TYPE(TDataType, value)->GetSchemeType() == expectedType, + // TODO: support pg types + MKQL_ENSURE(expectedType.GetTypeId() != NScheme::NTypeIds::Pg, "pg types are not supported"); + MKQL_ENSURE(AS_TYPE(TDataType, value)->GetSchemeType() == expectedType.GetTypeId(), "Mismatch of column type"); TDataType* byteType = TDataType::Create(NUdf::TDataType<ui8>::Id, Env); std::array<TType*, 2> elements; @@ -199,19 +207,21 @@ TRuntimeNode TKikimrProgramBuilder::RewriteNullType( } TVector<TRuntimeNode> TKikimrProgramBuilder::FixKeysType( - const TArrayRef<const ui32>& keyTypes, + const TArrayRef<NScheme::TTypeInfo>& keyTypes, const TKeyColumnValues& row) const { MKQL_ENSURE(!row.empty(), "Expected at least 1 key column"); MKQL_ENSURE(keyTypes.size() == row.size(), "Mismatch of key types count"); TVector<TRuntimeNode> tmp(row.size()); for (ui32 i = 0; i < row.size(); ++i) { - tmp[i] = RewriteNullType(row[i], keyTypes[i]); + // TODO: support pg types + MKQL_ENSURE(keyTypes[i].GetTypeId() != NScheme::NTypeIds::Pg, "pg types are not supported"); + tmp[i] = RewriteNullType(row[i], keyTypes[i].GetTypeId()); bool isOptional; TDataType* dataType = UnpackOptionalData(tmp[i], isOptional); - MKQL_ENSURE(keyTypes[i] == dataType->GetSchemeType(), + MKQL_ENSURE(keyTypes[i].GetTypeId() == dataType->GetSchemeType(), "Mismatch of key column type, expected: " - << keyTypes[i] << ", but got: " << dataType->GetSchemeType()); + << keyTypes[i].GetTypeId() << ", but got: " << dataType->GetSchemeType()); } return tmp; } @@ -223,7 +233,7 @@ TRuntimeNode TKikimrProgramBuilder::ReadTarget(const TReadTarget& target) { TRuntimeNode TKikimrProgramBuilder::SelectRow( const TTableId& tableId, - const TArrayRef<const ui32>& keyTypes, + const TArrayRef<NScheme::TTypeInfo>& keyTypes, const TArrayRef<const TSelectColumn>& columns, const TKeyColumnValues& row, const TReadTarget& target) { @@ -232,7 +242,7 @@ TRuntimeNode TKikimrProgramBuilder::SelectRow( TRuntimeNode TKikimrProgramBuilder::SelectRow( const TTableId& tableId, - const TArrayRef<const ui32>& keyTypes, + const TArrayRef<NScheme::TTypeInfo>& keyTypes, const TArrayRef<const TSelectColumn>& columns, const TKeyColumnValues& row, TRuntimeNode readTarget) { @@ -258,7 +268,7 @@ TRuntimeNode TKikimrProgramBuilder::SelectRow( TRuntimeNode TKikimrProgramBuilder::SelectRange( const TTableId& tableId, - const TArrayRef<const ui32>& keyTypes, + const TArrayRef<NScheme::TTypeInfo>& keyTypes, const TArrayRef<const TSelectColumn>& columns, const TTableRangeOptions& options, const TReadTarget& target) @@ -284,7 +294,7 @@ static bool AtLeastOneFlagSet(const TArrayRef<bool>& arr) { TRuntimeNode TKikimrProgramBuilder::SelectRange( const TTableId& tableId, - const TArrayRef<const ui32>& keyTypes, + const TArrayRef<NScheme::TTypeInfo>& keyTypes, const TArrayRef<const TSelectColumn>& columns, const TTableRangeOptions& options, TRuntimeNode readTarget) @@ -307,14 +317,16 @@ TRuntimeNode TKikimrProgramBuilder::SelectRange( TDataType* dataFrom = nullptr; TDataType* dataTo = nullptr; if (i < options.FromColumns.size()) { - from[i] = RewriteNullType(options.FromColumns[i], keyTypes[i]); + MKQL_ENSURE(keyTypes[i].GetTypeId() != NScheme::NTypeIds::Pg, "pg types are not supported"); + from[i] = RewriteNullType(options.FromColumns[i], keyTypes[i].GetTypeId()); bool isOptionalFrom; dataFrom = UnpackOptionalData(from[i], isOptionalFrom); } if (i < options.ToColumns.size()) { - to[i] = RewriteNullType(options.ToColumns[i], keyTypes[i]); + MKQL_ENSURE(keyTypes[i].GetTypeId() != NScheme::NTypeIds::Pg, "pg types are not supported"); + to[i] = RewriteNullType(options.ToColumns[i], keyTypes[i].GetTypeId()); bool isOptionalTo; dataTo = UnpackOptionalData(to[i], isOptionalTo); @@ -329,8 +341,8 @@ TRuntimeNode TKikimrProgramBuilder::SelectRange( } ui32 dataType = dataFrom ? dataFrom->GetSchemeType() : dataTo->GetSchemeType(); - MKQL_ENSURE(keyTypes[i] == dataType, "Mismatch of key column " << i - << " type, expected: " << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(keyTypes[i])).Name + MKQL_ENSURE(keyTypes[i].GetTypeId() == dataType, "Mismatch of key column " << i + << " type, expected: " << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(keyTypes[i].GetTypeId())).Name << ", but got: " << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(dataType)).Name); } @@ -375,7 +387,7 @@ TRuntimeNode TKikimrProgramBuilder::SelectRange( TRuntimeNode TKikimrProgramBuilder::UpdateRow( const TTableId& tableId, - const TArrayRef<const ui32>& keyTypes, + const TArrayRef<NScheme::TTypeInfo>& keyTypes, const TKeyColumnValues& row, TUpdateRowBuilder& update) { @@ -391,7 +403,7 @@ TRuntimeNode TKikimrProgramBuilder::UpdateRow( TRuntimeNode TKikimrProgramBuilder::EraseRow( const TTableId& tableId, - const TArrayRef<const ui32>& keyTypes, + const TArrayRef<NScheme::TTypeInfo>& keyTypes, const TKeyColumnValues& row) { auto rows = FixKeysType(keyTypes, row); diff --git a/ydb/core/engine/kikimr_program_builder.h b/ydb/core/engine/kikimr_program_builder.h index 446d976dd3e..267f06bdaeb 100644 --- a/ydb/core/engine/kikimr_program_builder.h +++ b/ydb/core/engine/kikimr_program_builder.h @@ -13,12 +13,12 @@ static constexpr NUdf::TDataTypeId LegacyPairUi64Ui64 = 0x101; struct TSelectColumn { TStringBuf Label; ui32 ColumnId = 0; - NUdf::TDataTypeId SchemeType = 0; + NScheme::TTypeInfo SchemeType; EColumnTypeConstraint TypeConstraint = EColumnTypeConstraint::Nullable; TSelectColumn() {} - TSelectColumn(TStringBuf label, ui32 columnId, NUdf::TDataTypeId schemeType, EColumnTypeConstraint typeConstraint) + TSelectColumn(TStringBuf label, ui32 columnId, NScheme::TTypeInfo schemeType, EColumnTypeConstraint typeConstraint) : Label(label) , ColumnId(columnId) , SchemeType(schemeType) @@ -91,10 +91,10 @@ public: TUpdateRowBuilder(const TUpdateRowBuilder&) = default; TUpdateRowBuilder& operator=(const TUpdateRowBuilder&) = default; // Supports Data or Optional of Data - void SetColumn(ui32 columnId, ui32 expectedType, TRuntimeNode value); + void SetColumn(ui32 columnId, NScheme::TTypeInfo expectedType, TRuntimeNode value); // Supports Data void InplaceUpdateColumn( - ui32 columnId, ui32 expectedType, + ui32 columnId, NScheme::TTypeInfo expectedType, TRuntimeNode value, EInplaceUpdateMode mode); void EraseColumn(ui32 columnId); TRuntimeNode Build(); @@ -138,14 +138,14 @@ public: // returns Optional of structType TRuntimeNode SelectRow( const TTableId& tableId, - const TArrayRef<const ui32>& keyTypes, + const TArrayRef<NScheme::TTypeInfo>& keyTypes, const TArrayRef<const TSelectColumn>& columns, const TKeyColumnValues& row, const TReadTarget& target = TReadTarget()); TRuntimeNode SelectRow( const TTableId& tableId, - const TArrayRef<const ui32>& keyTypes, + const TArrayRef<NScheme::TTypeInfo>& keyTypes, const TArrayRef<const TSelectColumn>& columns, const TKeyColumnValues& row, TRuntimeNode readTarget); @@ -158,14 +158,14 @@ public: // returns Struct with fields { List: List of structType, Truncated: Bool } TRuntimeNode SelectRange( const TTableId& tableId, - const TArrayRef<const ui32>& keyTypes, + const TArrayRef<NScheme::TTypeInfo>& keyTypes, const TArrayRef<const TSelectColumn>& columns, const TTableRangeOptions& options, const TReadTarget& target = TReadTarget()); TRuntimeNode SelectRange( const TTableId& tableId, - const TArrayRef<const ui32>& keyTypes, + const TArrayRef<NScheme::TTypeInfo>& keyTypes, const TArrayRef<const TSelectColumn>& columns, const TTableRangeOptions& options, TRuntimeNode readTarget); @@ -177,14 +177,14 @@ public: // returns Void TRuntimeNode UpdateRow( const TTableId& tableId, - const TArrayRef<const ui32>& keyTypes, + const TArrayRef<NScheme::TTypeInfo>& keyTypes, const TKeyColumnValues& row, TUpdateRowBuilder& update); // returns Void TRuntimeNode EraseRow( const TTableId& tableId, - const TArrayRef<const ui32>& keyTypes, + const TArrayRef<NScheme::TTypeInfo>& keyTypes, const TKeyColumnValues& row); //-- parameters functions @@ -236,7 +236,7 @@ private: TRuntimeNode NewDataLiteral(const std::pair<ui64, ui64>& data) const; TRuntimeNode BuildTableId(const TTableId& tableId) const; TVector<TRuntimeNode> FixKeysType( - const TArrayRef<const ui32>& keyTypes, + const TArrayRef<NScheme::TTypeInfo>& keyTypes, const TKeyColumnValues& row) const; TRuntimeNode RewriteNullType( TRuntimeNode value, diff --git a/ydb/core/engine/kikimr_program_builder_ut.cpp b/ydb/core/engine/kikimr_program_builder_ut.cpp index 00c58c32b7b..4173d753c80 100644 --- a/ydb/core/engine/kikimr_program_builder_ut.cpp +++ b/ydb/core/engine/kikimr_program_builder_ut.cpp @@ -16,6 +16,9 @@ namespace NKikimr { namespace NMiniKQL { + + using TTypeInfo = NScheme::TTypeInfo; + TIntrusivePtr<IRandomProvider> CreateRandomProvider() { return CreateDeterministicRandomProvider(1); } @@ -91,7 +94,11 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { keyColumns.push_back(pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui64>::Id)); keyColumns.push_back(pgmBuilder.NewOptional( pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("qwe"))); - TVector<ui32> keyTypes({ NUdf::TDataType<ui32>::Id, NUdf::TDataType<ui64>::Id, NUdf::TDataType<char*>::Id }); + TVector<TTypeInfo> keyTypes({ + TTypeInfo(NUdf::TDataType<ui32>::Id), + TTypeInfo(NUdf::TDataType<ui64>::Id), + TTypeInfo(NUdf::TDataType<char*>::Id) + }); pgmReturn = pgmBuilder.Append(pgmReturn, pgmBuilder.EraseRow(TTableId(1, 2), keyTypes, keyColumns)); @@ -118,9 +125,9 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { UNIT_ASSERT(tableKeys[0]->Range.To[2].Size() == 3); UNIT_ASSERT(tableKeys[0]->Columns.size() == 0); UNIT_ASSERT(tableKeys[0]->KeyColumnTypes.size() == 3); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0] == NUdf::TDataType<ui32>::Id); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1] == NUdf::TDataType<ui64>::Id); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[2] == NUdf::TDataType<char*>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0].GetTypeId() == NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1].GetTypeId() == NUdf::TDataType<ui64>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[2].GetTypeId() == NUdf::TDataType<char*>::Id); } Y_UNIT_TEST(TestEraseRowPartialDynamicKey) { @@ -135,7 +142,11 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(34), pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(12))); keyColumns.push_back(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("qwe")); - TVector<ui32> keyTypes({ NUdf::TDataType<ui64>::Id, NUdf::TDataType<ui32>::Id, NUdf::TDataType<char*>::Id }); + TVector<TTypeInfo> keyTypes({ + TTypeInfo(NUdf::TDataType<ui64>::Id), + TTypeInfo(NUdf::TDataType<ui32>::Id), + TTypeInfo(NUdf::TDataType<char*>::Id) + }); pgmReturn = pgmBuilder.Append(pgmReturn, pgmBuilder.EraseRow(TTableId(1, 2), keyTypes, keyColumns)); @@ -160,9 +171,9 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { UNIT_ASSERT(tableKeys[0]->Range.To[0].IsNull()); UNIT_ASSERT(tableKeys[0]->Columns.size() == 0); UNIT_ASSERT(tableKeys[0]->KeyColumnTypes.size() == 3); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0] == NUdf::TDataType<ui64>::Id); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1] == NUdf::TDataType<ui32>::Id); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[2] == NUdf::TDataType<char*>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0].GetTypeId() == NUdf::TDataType<ui64>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1].GetTypeId() == NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[2].GetTypeId() == NUdf::TDataType<char*>::Id); } Y_UNIT_TEST(TestEraseRowDynamicKey) { @@ -177,7 +188,10 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(12))); keyColumns.push_back(pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("qwe")); - TVector<ui32> keyTypes({ NUdf::TDataType<ui32>::Id, NUdf::TDataType<char*>::Id }); + TVector<TTypeInfo> keyTypes({ + TTypeInfo(NUdf::TDataType<ui32>::Id), + TTypeInfo(NUdf::TDataType<char*>::Id) + }); pgmReturn = pgmBuilder.Append(pgmReturn, pgmBuilder.EraseRow(TTableId(1, 2), keyTypes, keyColumns)); @@ -200,8 +214,8 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { UNIT_ASSERT(tableKeys[0]->Range.To.size() == 0); UNIT_ASSERT(tableKeys[0]->Columns.size() == 0); UNIT_ASSERT(tableKeys[0]->KeyColumnTypes.size() == 2); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0] == NUdf::TDataType<ui32>::Id); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1] == NUdf::TDataType<char*>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0].GetTypeId() == NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1].GetTypeId() == NUdf::TDataType<char*>::Id); } Y_UNIT_TEST(TestSelectRow) { @@ -216,9 +230,13 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { keyColumns.push_back(pgmBuilder.NewOptional( pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("qwe"))); TVector<TSelectColumn> columnsToRead; - columnsToRead.emplace_back("column1", 34, (ui32)NUdf::TDataType<ui32>::Id, EColumnTypeConstraint::Nullable); - columnsToRead.emplace_back("column2", 56, (ui32)NUdf::TDataType<ui64>::Id, EColumnTypeConstraint::Nullable); - TVector<ui32> keyTypes({ NUdf::TDataType<ui32>::Id, NUdf::TDataType<ui64>::Id, NUdf::TDataType<char*>::Id }); + columnsToRead.emplace_back("column1", 34, TTypeInfo(NUdf::TDataType<ui32>::Id), EColumnTypeConstraint::Nullable); + columnsToRead.emplace_back("column2", 56, TTypeInfo(NUdf::TDataType<ui64>::Id), EColumnTypeConstraint::Nullable); + TVector<TTypeInfo> keyTypes({ + TTypeInfo(NUdf::TDataType<ui32>::Id), + TTypeInfo(NUdf::TDataType<ui64>::Id), + TTypeInfo(NUdf::TDataType<char*>::Id) + }); pgmReturn = pgmBuilder.Append(pgmReturn, pgmBuilder.SetResult("myRes", pgmBuilder.SelectRow(TTableId(1, 2), keyTypes, @@ -247,14 +265,14 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { UNIT_ASSERT(tableKeys[0]->Columns.size() == 2); UNIT_ASSERT(tableKeys[0]->Columns[0].Column == 34); UNIT_ASSERT(tableKeys[0]->Columns[0].Operation == TKeyDesc::EColumnOperation::Read); - UNIT_ASSERT(tableKeys[0]->Columns[0].ExpectedType == NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(tableKeys[0]->Columns[0].ExpectedType.GetTypeId() == NUdf::TDataType<ui32>::Id); UNIT_ASSERT(tableKeys[0]->Columns[1].Column == 56); UNIT_ASSERT(tableKeys[0]->Columns[1].Operation == TKeyDesc::EColumnOperation::Read); - UNIT_ASSERT(tableKeys[0]->Columns[1].ExpectedType == NUdf::TDataType<ui64>::Id); + UNIT_ASSERT(tableKeys[0]->Columns[1].ExpectedType.GetTypeId() == NUdf::TDataType<ui64>::Id); UNIT_ASSERT(tableKeys[0]->KeyColumnTypes.size() == 3); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0] == NUdf::TDataType<ui32>::Id); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1] == NUdf::TDataType<ui64>::Id); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[2] == NUdf::TDataType<char*>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0].GetTypeId() == NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1].GetTypeId() == NUdf::TDataType<ui64>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[2].GetTypeId() == NUdf::TDataType<char*>::Id); } Y_UNIT_TEST(TestUpdateRowStaticKey) { @@ -269,12 +287,16 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { keyColumns.push_back(pgmBuilder.NewOptional( pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("qwe"))); auto update = pgmBuilder.GetUpdateRowBuilder(); - update.SetColumn(34, NUdf::TDataType<ui32>::Id, + update.SetColumn(34, TTypeInfo(NUdf::TDataType<ui32>::Id), pgmBuilder.NewOptional(pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(12))); update.EraseColumn(56); - update.InplaceUpdateColumn(78, NUdf::TDataType<ui64>::Id, + update.InplaceUpdateColumn(78, TTypeInfo(NUdf::TDataType<ui64>::Id), pgmBuilder.TProgramBuilder::NewDataLiteral<ui64>(1), EInplaceUpdateMode::Sum); - TVector<ui32> keyTypes({ NUdf::TDataType<ui32>::Id, NUdf::TDataType<ui64>::Id, NUdf::TDataType<char*>::Id }); + TVector<TTypeInfo> keyTypes({ + TTypeInfo(NUdf::TDataType<ui32>::Id), + TTypeInfo(NUdf::TDataType<ui64>::Id), + TTypeInfo(NUdf::TDataType<char*>::Id) + }); pgmReturn = pgmBuilder.Append(pgmReturn, pgmBuilder.UpdateRow(TTableId(1, 2), keyTypes, keyColumns, update)); @@ -302,17 +324,17 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { UNIT_ASSERT(tableKeys[0]->Columns.size() == 3); UNIT_ASSERT(tableKeys[0]->Columns[0].Column == 34); UNIT_ASSERT(tableKeys[0]->Columns[0].Operation == TKeyDesc::EColumnOperation::Set); - UNIT_ASSERT(tableKeys[0]->Columns[0].ExpectedType == NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(tableKeys[0]->Columns[0].ExpectedType.GetTypeId() == NUdf::TDataType<ui32>::Id); UNIT_ASSERT(tableKeys[0]->Columns[1].Column == 56); UNIT_ASSERT(tableKeys[0]->Columns[1].Operation == TKeyDesc::EColumnOperation::Set); - UNIT_ASSERT(tableKeys[0]->Columns[1].ExpectedType == 0); + UNIT_ASSERT(tableKeys[0]->Columns[1].ExpectedType.GetTypeId() == 0); UNIT_ASSERT(tableKeys[0]->Columns[2].Column == 78); UNIT_ASSERT(tableKeys[0]->Columns[2].Operation == TKeyDesc::EColumnOperation::InplaceUpdate); - UNIT_ASSERT(tableKeys[0]->Columns[2].ExpectedType == NUdf::TDataType<ui64>::Id); + UNIT_ASSERT(tableKeys[0]->Columns[2].ExpectedType.GetTypeId() == NUdf::TDataType<ui64>::Id); UNIT_ASSERT(tableKeys[0]->KeyColumnTypes.size() == 3); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0] == NUdf::TDataType<ui32>::Id); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1] == NUdf::TDataType<ui64>::Id); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[2] == NUdf::TDataType<char*>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0].GetTypeId() == NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1].GetTypeId() == NUdf::TDataType<ui64>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[2].GetTypeId() == NUdf::TDataType<char*>::Id); } Y_UNIT_TEST(TestUpdateRowDynamicKey) { @@ -329,12 +351,16 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { keyColumns.push_back(pgmBuilder.NewOptional( pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("qwe"))); auto update = pgmBuilder.GetUpdateRowBuilder(); - update.SetColumn(34, NUdf::TDataType<ui32>::Id, + update.SetColumn(34, TTypeInfo(NUdf::TDataType<ui32>::Id), pgmBuilder.NewOptional(pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(12))); update.EraseColumn(56); - update.InplaceUpdateColumn(78, NUdf::TDataType<ui64>::Id, + update.InplaceUpdateColumn(78, TTypeInfo(NUdf::TDataType<ui64>::Id), pgmBuilder.TProgramBuilder::NewDataLiteral<ui64>(1), EInplaceUpdateMode::Sum); - TVector<ui32> keyTypes({ NUdf::TDataType<ui32>::Id, NUdf::TDataType<ui64>::Id, NUdf::TDataType<char*>::Id }); + TVector<TTypeInfo> keyTypes({ + TTypeInfo(NUdf::TDataType<ui32>::Id), + TTypeInfo(NUdf::TDataType<ui64>::Id), + TTypeInfo(NUdf::TDataType<char*>::Id) + }); pgmReturn = pgmBuilder.Append(pgmReturn, pgmBuilder.UpdateRow(TTableId(1, 2), keyTypes, keyColumns, update)); @@ -359,18 +385,18 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { UNIT_ASSERT(tableKeys[0]->Columns.size() == 3); UNIT_ASSERT(tableKeys[0]->Columns[0].Column == 34); UNIT_ASSERT(tableKeys[0]->Columns[0].Operation == TKeyDesc::EColumnOperation::Set); - UNIT_ASSERT(tableKeys[0]->Columns[0].ExpectedType == NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(tableKeys[0]->Columns[0].ExpectedType.GetTypeId() == NUdf::TDataType<ui32>::Id); UNIT_ASSERT(tableKeys[0]->Columns[1].Column == 56); UNIT_ASSERT(tableKeys[0]->Columns[1].Operation == TKeyDesc::EColumnOperation::Set); - UNIT_ASSERT(tableKeys[0]->Columns[1].ExpectedType == 0); + UNIT_ASSERT(tableKeys[0]->Columns[1].ExpectedType.GetTypeId() == 0); UNIT_ASSERT(tableKeys[0]->Columns[2].Column == 78); UNIT_ASSERT(tableKeys[0]->Columns[2].Operation == TKeyDesc::EColumnOperation::InplaceUpdate); - UNIT_ASSERT(tableKeys[0]->Columns[2].ExpectedType == NUdf::TDataType<ui64>::Id); + UNIT_ASSERT(tableKeys[0]->Columns[2].ExpectedType.GetTypeId() == NUdf::TDataType<ui64>::Id); UNIT_ASSERT(tableKeys[0]->Columns[2].InplaceUpdateMode == (ui32)EInplaceUpdateMode::Sum); UNIT_ASSERT(tableKeys[0]->KeyColumnTypes.size() == 3); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0] == NUdf::TDataType<ui32>::Id); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1] == NUdf::TDataType<ui64>::Id); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[2] == NUdf::TDataType<char*>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0].GetTypeId() == NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1].GetTypeId() == NUdf::TDataType<ui64>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[2].GetTypeId() == NUdf::TDataType<char*>::Id); } Y_UNIT_TEST(TestSelectFromInclusiveRange) { @@ -386,9 +412,12 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { options.FromColumns = from; TVector<TSelectColumn> columnsToRead; - columnsToRead.emplace_back("column1", 34, (ui32)NUdf::TDataType<ui32>::Id, EColumnTypeConstraint::Nullable); - columnsToRead.emplace_back("column2", 56, (ui32)NUdf::TDataType<ui64>::Id, EColumnTypeConstraint::Nullable); - TVector<ui32> keyTypes({ NUdf::TDataType<ui32>::Id, NUdf::TDataType<ui64>::Id }); + columnsToRead.emplace_back("column1", 34, TTypeInfo(NUdf::TDataType<ui32>::Id), EColumnTypeConstraint::Nullable); + columnsToRead.emplace_back("column2", 56, TTypeInfo(NUdf::TDataType<ui64>::Id), EColumnTypeConstraint::Nullable); + TVector<TTypeInfo> keyTypes({ + TTypeInfo(NUdf::TDataType<ui32>::Id), + TTypeInfo(NUdf::TDataType<ui64>::Id) + }); pgmReturn = pgmBuilder.Append(pgmReturn, pgmBuilder.SetResult("myRes", pgmBuilder.SelectRange(TTableId(1, 2), keyTypes, @@ -411,15 +440,15 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { UNIT_ASSERT(tableKeys[0]->Range.From[1].IsNull()); UNIT_ASSERT(tableKeys[0]->Range.To.size() == 0); UNIT_ASSERT(tableKeys[0]->KeyColumnTypes.size() == 2); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0] == NUdf::TDataType<ui32>::Id); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1] == NUdf::TDataType<ui64>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0].GetTypeId() == NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1].GetTypeId() == NUdf::TDataType<ui64>::Id); UNIT_ASSERT(tableKeys[0]->Columns.size() == 2); UNIT_ASSERT(tableKeys[0]->Columns[0].Column == 34); UNIT_ASSERT(tableKeys[0]->Columns[0].Operation == TKeyDesc::EColumnOperation::Read); - UNIT_ASSERT(tableKeys[0]->Columns[0].ExpectedType == NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(tableKeys[0]->Columns[0].ExpectedType.GetTypeId() == NUdf::TDataType<ui32>::Id); UNIT_ASSERT(tableKeys[0]->Columns[1].Column == 56); UNIT_ASSERT(tableKeys[0]->Columns[1].Operation == TKeyDesc::EColumnOperation::Read); - UNIT_ASSERT(tableKeys[0]->Columns[1].ExpectedType == NUdf::TDataType<ui64>::Id); + UNIT_ASSERT(tableKeys[0]->Columns[1].ExpectedType.GetTypeId() == NUdf::TDataType<ui64>::Id); } Y_UNIT_TEST(TestSelectFromExclusiveRange) { @@ -436,9 +465,12 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { options.Flags = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(TReadRangeOptions::TFlags::ExcludeInitValue); TVector<TSelectColumn> columnsToRead; - columnsToRead.emplace_back("column1", 34, (ui32)NUdf::TDataType<ui32>::Id, EColumnTypeConstraint::Nullable); - columnsToRead.emplace_back("column2", 56, (ui32)NUdf::TDataType<ui64>::Id, EColumnTypeConstraint::Nullable); - TVector<ui32> keyTypes({ NUdf::TDataType<ui32>::Id, NUdf::TDataType<ui64>::Id }); + columnsToRead.emplace_back("column1", 34, TTypeInfo(NUdf::TDataType<ui32>::Id), EColumnTypeConstraint::Nullable); + columnsToRead.emplace_back("column2", 56, TTypeInfo(NUdf::TDataType<ui64>::Id), EColumnTypeConstraint::Nullable); + TVector<TTypeInfo> keyTypes({ + TTypeInfo(NUdf::TDataType<ui32>::Id), + TTypeInfo(NUdf::TDataType<ui64>::Id) + }); pgmReturn = pgmBuilder.Append(pgmReturn, pgmBuilder.SetResult("myRes", pgmBuilder.SelectRange(TTableId(1, 2), keyTypes, @@ -461,15 +493,15 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { UNIT_ASSERT(tableKeys[0]->Range.From[1].IsNull()); UNIT_ASSERT(tableKeys[0]->Range.To.size() == 0); UNIT_ASSERT(tableKeys[0]->KeyColumnTypes.size() == 2); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0] == NUdf::TDataType<ui32>::Id); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1] == NUdf::TDataType<ui64>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0].GetTypeId() == NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1].GetTypeId() == NUdf::TDataType<ui64>::Id); UNIT_ASSERT(tableKeys[0]->Columns.size() == 2); UNIT_ASSERT(tableKeys[0]->Columns[0].Column == 34); UNIT_ASSERT(tableKeys[0]->Columns[0].Operation == TKeyDesc::EColumnOperation::Read); - UNIT_ASSERT(tableKeys[0]->Columns[0].ExpectedType == NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(tableKeys[0]->Columns[0].ExpectedType.GetTypeId() == NUdf::TDataType<ui32>::Id); UNIT_ASSERT(tableKeys[0]->Columns[1].Column == 56); UNIT_ASSERT(tableKeys[0]->Columns[1].Operation == TKeyDesc::EColumnOperation::Read); - UNIT_ASSERT(tableKeys[0]->Columns[1].ExpectedType == NUdf::TDataType<ui64>::Id); + UNIT_ASSERT(tableKeys[0]->Columns[1].ExpectedType.GetTypeId() == NUdf::TDataType<ui64>::Id); } Y_UNIT_TEST(TestSelectToInclusiveRange) { @@ -488,9 +520,12 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { options.ToColumns = to; TVector<TSelectColumn> columnsToRead; - columnsToRead.emplace_back("column1", 34, (ui32)NUdf::TDataType<ui32>::Id, EColumnTypeConstraint::Nullable); - columnsToRead.emplace_back("column2", 56, (ui32)NUdf::TDataType<ui64>::Id, EColumnTypeConstraint::Nullable); - TVector<ui32> keyTypes({ NUdf::TDataType<ui32>::Id, NUdf::TDataType<ui64>::Id }); + columnsToRead.emplace_back("column1", 34, TTypeInfo(NUdf::TDataType<ui32>::Id), EColumnTypeConstraint::Nullable); + columnsToRead.emplace_back("column2", 56, TTypeInfo(NUdf::TDataType<ui64>::Id), EColumnTypeConstraint::Nullable); + TVector<TTypeInfo> keyTypes({ + TTypeInfo(NUdf::TDataType<ui32>::Id), + TTypeInfo(NUdf::TDataType<ui64>::Id) + }); pgmReturn = pgmBuilder.Append(pgmReturn, pgmBuilder.SetResult("myRes", pgmBuilder.SelectRange(TTableId(1, 2), keyTypes, @@ -515,15 +550,15 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { UNIT_ASSERT(tableKeys[0]->Range.To[0].Size() == 4); UNIT_ASSERT(tableKeys[0]->Range.To[1].IsNull()); UNIT_ASSERT(tableKeys[0]->KeyColumnTypes.size() == 2); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0] == NUdf::TDataType<ui32>::Id); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1] == NUdf::TDataType<ui64>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0].GetTypeId() == NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1].GetTypeId() == NUdf::TDataType<ui64>::Id); UNIT_ASSERT(tableKeys[0]->Columns.size() == 2); UNIT_ASSERT(tableKeys[0]->Columns[0].Column == 34); UNIT_ASSERT(tableKeys[0]->Columns[0].Operation == TKeyDesc::EColumnOperation::Read); - UNIT_ASSERT(tableKeys[0]->Columns[0].ExpectedType == NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(tableKeys[0]->Columns[0].ExpectedType.GetTypeId() == NUdf::TDataType<ui32>::Id); UNIT_ASSERT(tableKeys[0]->Columns[1].Column == 56); UNIT_ASSERT(tableKeys[0]->Columns[1].Operation == TKeyDesc::EColumnOperation::Read); - UNIT_ASSERT(tableKeys[0]->Columns[1].ExpectedType == NUdf::TDataType<ui64>::Id); + UNIT_ASSERT(tableKeys[0]->Columns[1].ExpectedType.GetTypeId() == NUdf::TDataType<ui64>::Id); } Y_UNIT_TEST(TestSelectToExclusiveRange) { @@ -543,9 +578,12 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { options.Flags = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(TReadRangeOptions::TFlags::ExcludeTermValue); TVector<TSelectColumn> columnsToRead; - columnsToRead.emplace_back("column1", 34, (ui32)NUdf::TDataType<ui32>::Id, EColumnTypeConstraint::Nullable); - columnsToRead.emplace_back("column2", 56, (ui32)NUdf::TDataType<ui64>::Id, EColumnTypeConstraint::Nullable); - TVector<ui32> keyTypes({ NUdf::TDataType<ui32>::Id, NUdf::TDataType<ui64>::Id }); + columnsToRead.emplace_back("column1", 34, TTypeInfo(NUdf::TDataType<ui32>::Id), EColumnTypeConstraint::Nullable); + columnsToRead.emplace_back("column2", 56, TTypeInfo(NUdf::TDataType<ui64>::Id), EColumnTypeConstraint::Nullable); + TVector<TTypeInfo> keyTypes({ + TTypeInfo(NUdf::TDataType<ui32>::Id), + TTypeInfo(NUdf::TDataType<ui64>::Id) + }); pgmReturn = pgmBuilder.Append(pgmReturn, pgmBuilder.SetResult("myRes", pgmBuilder.SelectRange(TTableId(1, 2), keyTypes, @@ -570,15 +608,15 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { UNIT_ASSERT(tableKeys[0]->Range.To[0].Size() == 4); UNIT_ASSERT(tableKeys[0]->Range.To[1].IsNull()); UNIT_ASSERT(tableKeys[0]->KeyColumnTypes.size() == 2); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0] == NUdf::TDataType<ui32>::Id); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1] == NUdf::TDataType<ui64>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0].GetTypeId() == NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[1].GetTypeId() == NUdf::TDataType<ui64>::Id); UNIT_ASSERT(tableKeys[0]->Columns.size() == 2); UNIT_ASSERT(tableKeys[0]->Columns[0].Column == 34); UNIT_ASSERT(tableKeys[0]->Columns[0].Operation == TKeyDesc::EColumnOperation::Read); - UNIT_ASSERT(tableKeys[0]->Columns[0].ExpectedType == NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(tableKeys[0]->Columns[0].ExpectedType.GetTypeId() == NUdf::TDataType<ui32>::Id); UNIT_ASSERT(tableKeys[0]->Columns[1].Column == 56); UNIT_ASSERT(tableKeys[0]->Columns[1].Operation == TKeyDesc::EColumnOperation::Read); - UNIT_ASSERT(tableKeys[0]->Columns[1].ExpectedType == NUdf::TDataType<ui64>::Id); + UNIT_ASSERT(tableKeys[0]->Columns[1].ExpectedType.GetTypeId() == NUdf::TDataType<ui64>::Id); } Y_UNIT_TEST(TestSelectBothFromInclusiveToInclusiveRange) { @@ -596,9 +634,9 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { options.ToColumns = to; TVector<TSelectColumn> columnsToRead; - columnsToRead.emplace_back("column1", 34, (ui32)NUdf::TDataType<ui32>::Id, EColumnTypeConstraint::Nullable); - columnsToRead.emplace_back("column2", 56, (ui32)NUdf::TDataType<ui64>::Id, EColumnTypeConstraint::Nullable); - TVector<ui32> keyTypes({ NUdf::TDataType<char*>::Id }); + columnsToRead.emplace_back("column1", 34, TTypeInfo(NUdf::TDataType<ui32>::Id), EColumnTypeConstraint::Nullable); + columnsToRead.emplace_back("column2", 56, TTypeInfo(NUdf::TDataType<ui64>::Id), EColumnTypeConstraint::Nullable); + TVector<TTypeInfo> keyTypes({ TTypeInfo(NUdf::TDataType<char*>::Id) }); pgmReturn = pgmBuilder.Append(pgmReturn, pgmBuilder.SetResult("myRes", pgmBuilder.SelectRange(TTableId(1, 2), keyTypes, @@ -621,14 +659,14 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { UNIT_ASSERT(tableKeys[0]->Range.To.size() == 1); UNIT_ASSERT(tableKeys[0]->Range.To[0].Size() == 2); UNIT_ASSERT(tableKeys[0]->KeyColumnTypes.size() == 1); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0] == NUdf::TDataType<char*>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0].GetTypeId() == NUdf::TDataType<char*>::Id); UNIT_ASSERT(tableKeys[0]->Columns.size() == 2); UNIT_ASSERT(tableKeys[0]->Columns[0].Column == 34); UNIT_ASSERT(tableKeys[0]->Columns[0].Operation == TKeyDesc::EColumnOperation::Read); - UNIT_ASSERT(tableKeys[0]->Columns[0].ExpectedType == NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(tableKeys[0]->Columns[0].ExpectedType.GetTypeId() == NUdf::TDataType<ui32>::Id); UNIT_ASSERT(tableKeys[0]->Columns[1].Column == 56); UNIT_ASSERT(tableKeys[0]->Columns[1].Operation == TKeyDesc::EColumnOperation::Read); - UNIT_ASSERT(tableKeys[0]->Columns[1].ExpectedType == NUdf::TDataType<ui64>::Id); + UNIT_ASSERT(tableKeys[0]->Columns[1].ExpectedType.GetTypeId() == NUdf::TDataType<ui64>::Id); } Y_UNIT_TEST(TestSelectBothFromExclusiveToExclusiveRange) { @@ -648,9 +686,9 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { TReadRangeOptions::TFlags::ExcludeInitValue | TReadRangeOptions::TFlags::ExcludeTermValue); TVector<TSelectColumn> columnsToRead; - columnsToRead.emplace_back("column1", 34, (ui32)NUdf::TDataType<ui32>::Id, EColumnTypeConstraint::Nullable); - columnsToRead.emplace_back("column2", 56, (ui32)NUdf::TDataType<ui64>::Id, EColumnTypeConstraint::Nullable); - TVector<ui32> keyTypes({ NUdf::TDataType<char*>::Id }); + columnsToRead.emplace_back("column1", 34, TTypeInfo(NUdf::TDataType<ui32>::Id), EColumnTypeConstraint::Nullable); + columnsToRead.emplace_back("column2", 56, TTypeInfo(NUdf::TDataType<ui64>::Id), EColumnTypeConstraint::Nullable); + TVector<TTypeInfo> keyTypes({ TTypeInfo(NUdf::TDataType<char*>::Id) }); pgmReturn = pgmBuilder.Append(pgmReturn, pgmBuilder.SetResult("myRes", pgmBuilder.SelectRange(TTableId(1, 2), keyTypes, @@ -673,14 +711,14 @@ Y_UNIT_TEST_SUITE(TMiniKQLProgramBuilderTest) { UNIT_ASSERT(tableKeys[0]->Range.To.size() == 1); UNIT_ASSERT(tableKeys[0]->Range.To[0].Size() == 2); UNIT_ASSERT(tableKeys[0]->KeyColumnTypes.size() == 1); - UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0] == NUdf::TDataType<char*>::Id); + UNIT_ASSERT(tableKeys[0]->KeyColumnTypes[0].GetTypeId() == NUdf::TDataType<char*>::Id); UNIT_ASSERT(tableKeys[0]->Columns.size() == 2); UNIT_ASSERT(tableKeys[0]->Columns[0].Column == 34); UNIT_ASSERT(tableKeys[0]->Columns[0].Operation == TKeyDesc::EColumnOperation::Read); - UNIT_ASSERT(tableKeys[0]->Columns[0].ExpectedType == NUdf::TDataType<ui32>::Id); + UNIT_ASSERT(tableKeys[0]->Columns[0].ExpectedType.GetTypeId() == NUdf::TDataType<ui32>::Id); UNIT_ASSERT(tableKeys[0]->Columns[1].Column == 56); UNIT_ASSERT(tableKeys[0]->Columns[1].Operation == TKeyDesc::EColumnOperation::Read); - UNIT_ASSERT(tableKeys[0]->Columns[1].ExpectedType == NUdf::TDataType<ui64>::Id); + UNIT_ASSERT(tableKeys[0]->Columns[1].ExpectedType.GetTypeId() == NUdf::TDataType<ui64>::Id); } Y_UNIT_TEST(TestAcquireLocks) { diff --git a/ydb/core/engine/minikql/flat_local_tx_read_columns.h b/ydb/core/engine/minikql/flat_local_tx_read_columns.h index 57d3c7bef9d..53e08fd2869 100644 --- a/ydb/core/engine/minikql/flat_local_tx_read_columns.h +++ b/ydb/core/engine/minikql/flat_local_tx_read_columns.h @@ -53,11 +53,11 @@ public: return true; } - TVector<NScheme::TTypeId> keyColumnTypes; + TVector<NScheme::TTypeInfo> keyColumnTypes; TSmallVec<TRawTypeValue> keyFrom; TSmallVec<TRawTypeValue> keyTo; - bool inclusiveFrom = Ev->Get()->Record.GetFromKeyInclusive();; - bool inclusiveTo = Ev->Get()->Record.GetToKeyInclusive();; + bool inclusiveFrom = Ev->Get()->Record.GetFromKeyInclusive(); + bool inclusiveTo = Ev->Get()->Record.GetToKeyInclusive(); // TODO: check schemas for (ui32 keyColId : tableInfo->KeyColumns) { @@ -78,7 +78,7 @@ public: } TVector<NTable::TTag> valueColumns; - TVector<std::pair<TString, NScheme::TTypeId>> columns; + TVector<std::pair<TString, NScheme::TTypeInfo>> columns; for (const auto& col : Ev->Get()->Record.GetColumns()) { const auto* colNameInfo = tableInfo->ColumnNames.FindPtr(col); if (!colNameInfo) { diff --git a/ydb/core/engine/minikql/minikql_engine_host.cpp b/ydb/core/engine/minikql/minikql_engine_host.cpp index d362d938068..84cf7fa3b21 100644 --- a/ydb/core/engine/minikql/minikql_engine_host.cpp +++ b/ydb/core/engine/minikql/minikql_engine_host.cpp @@ -22,7 +22,8 @@ void ConvertTableKeys(const TScheme& scheme, const TScheme::TTableInfo* tableInf for (size_t keyIdx = 0; keyIdx < row.size(); keyIdx++) { const TCell& cell = row[keyIdx]; ui32 keyCol = tableInfo->KeyColumns[keyIdx]; - NScheme::TTypeId vtype = scheme.GetColumnInfo(tableInfo, keyCol)->PType; + // TODO: support pg types + NScheme::TTypeId vtype = scheme.GetColumnInfo(tableInfo, keyCol)->PType.GetTypeId(); if (cell.IsNull()) { key.emplace_back(); bytes += 1; @@ -75,7 +76,7 @@ bool TEngineHost::IsValidKey(TKeyDesc& key, std::pair<ui64, ui64>& maxSnapshotTi // Specified keys types should be valid for any operation for (size_t keyIdx = 0; keyIdx < key.KeyColumnTypes.size(); keyIdx++) { ui32 keyCol = tableInfo->KeyColumns[keyIdx]; - NScheme::TTypeId vtype = Scheme.GetColumnInfo(tableInfo, keyCol)->PType; + auto vtype = Scheme.GetColumnInfo(tableInfo, keyCol)->PType; EH_VALIDATE(key.KeyColumnTypes[keyIdx] == vtype, TypeCheckFailed); } @@ -93,7 +94,7 @@ bool TEngineHost::IsValidKey(TKeyDesc& key, std::pair<ui64, ui64>& maxSnapshotTi } auto* cinfo = Scheme.GetColumnInfo(tableInfo, cop.Column); EH_VALIDATE(cinfo, TypeCheckFailed); // Unknown column - NScheme::TTypeId vtype = cinfo->PType; + auto vtype = cinfo->PType; EH_VALIDATE(cop.ExpectedType == vtype, TypeCheckFailed); EH_VALIDATE(cop.Operation == TKeyDesc::EColumnOperation::Read, OperationNotSupported); } @@ -103,8 +104,8 @@ bool TEngineHost::IsValidKey(TKeyDesc& key, std::pair<ui64, ui64>& maxSnapshotTi const TKeyDesc::TColumnOp& cop = key.Columns[i]; auto* cinfo = Scheme.GetColumnInfo(tableInfo, cop.Column); EH_VALIDATE(cinfo, TypeCheckFailed); // Unknown column - NScheme::TTypeId vtype = cinfo->PType; - EH_VALIDATE(!cop.ExpectedType || cop.ExpectedType == vtype, TypeCheckFailed); + auto vtype = cinfo->PType; + EH_VALIDATE(cop.ExpectedType.GetTypeId() == 0 || cop.ExpectedType == vtype, TypeCheckFailed); EH_VALIDATE(cop.Operation == TKeyDesc::EColumnOperation::Set, OperationNotSupported); // TODO[serxa]: support inplace operations in IsValidKey } } else if (key.RowOperation == TKeyDesc::ERowOperation::Erase) { @@ -286,7 +287,7 @@ NUdf::TUnboxedValue TEngineHost::SelectRow(const TTableId& tableId, const TArray TSmallVec<NTable::TTag> systemColumnTags; AnalyzeRowType(columnIds, tags, systemColumnTags); - TSmallVec<NScheme::TTypeId> cellTypes; + TSmallVec<NScheme::TTypeInfo> cellTypes; cellTypes.reserve(tags.size()); for (size_t i = 0; i < tags.size(); ++i) cellTypes.emplace_back(tableInfo->Columns.at(tags[i]).PType); @@ -538,8 +539,16 @@ public: TString firstKey; firstKey.reserve(totalSize); firstKey.AppendNoAlias((const char*)&tuple.ColumnCount, sizeof(ui32)); - firstKey.AppendNoAlias((const char*)tuple.Types, tuple.ColumnCount * sizeof(NScheme::TTypeId)); + TVector<NScheme::TTypeId> typeIds; + typeIds.reserve(tuple.ColumnCount); + for (ui32 i = 0; i < tuple.ColumnCount; ++i) { + auto typeId = tuple.Types[i].GetTypeId(); + Y_ENSURE(typeId != NScheme::NTypeIds::Pg, "pg types are not supported"); + typeIds.push_back(typeId); + } + firstKey.AppendNoAlias((const char*)typeIds.data(), tuple.ColumnCount * sizeof(NScheme::TTypeId)); firstKey.AppendNoAlias(cells); + // TODO: support pg types if (List.FirstKey) { Y_VERIFY_DEBUG(*List.FirstKey == firstKey); @@ -879,7 +888,8 @@ void TEngineHost::UpdateRow(const TTableId& tableId, const TArrayRef<const TCell for (size_t i = 0; i < commands.size(); i++) { const TUpdateCommand& upd = commands[i]; Y_VERIFY(upd.Operation == TKeyDesc::EColumnOperation::Set); // TODO[serxa]: support inplace update in update row - NScheme::TTypeId vtype = Scheme.GetColumnInfo(tableInfo, upd.Column)->PType; + // TODO: support pg types + NScheme::TTypeId vtype = Scheme.GetColumnInfo(tableInfo, upd.Column)->PType.GetTypeId(); ops.emplace_back(upd.Column, NTable::ECellOp::Set, upd.Value.IsNull() ? TRawTypeValue() : TRawTypeValue(upd.Value.Data(), upd.Value.Size(), vtype)); valueBytes += upd.Value.IsNull() ? 1 : upd.Value.Size(); @@ -996,12 +1006,12 @@ void AnalyzeRowType(TStructLiteral* columnIds, TSmallVec<NTable::TTag>& tags, TS } } -NUdf::TUnboxedValue GetCellValue(const TCell& cell, NScheme::TTypeId type) { +NUdf::TUnboxedValue GetCellValue(const TCell& cell, NScheme::TTypeInfo type) { if (cell.IsNull()) { return NUdf::TUnboxedValue(); } - switch (type) { + switch (type.GetTypeId()) { case NYql::NProto::TypeIds::Bool: return NUdf::TUnboxedValuePod(cell.AsValue<bool>()); case NYql::NProto::TypeIds::Int8: @@ -1065,9 +1075,16 @@ NUdf::TUnboxedValue GetCellValue(const TCell& cell, NScheme::TTypeId type) { return MakeString(NUdf::TStringRef(cell.Data(), cell.Size())); default: - Y_VERIFY_DEBUG(false, "Unsupported type: %" PRIu16, type); - return MakeString(NUdf::TStringRef(cell.Data(), cell.Size())); + break; } + + if (type.GetTypeId() == NScheme::NTypeIds::Pg) { + // TODO: support pg types + Y_VERIFY(false, "pg types are not supported"); + } + + Y_VERIFY_DEBUG(false, "Unsupported type: %" PRIu16, type.GetTypeId()); + return MakeString(NUdf::TStringRef(cell.Data(), cell.Size())); } }} diff --git a/ydb/core/engine/minikql/minikql_engine_host.h b/ydb/core/engine/minikql/minikql_engine_host.h index 7ae05b75237..7f8434eb887 100644 --- a/ydb/core/engine/minikql/minikql_engine_host.h +++ b/ydb/core/engine/minikql/minikql_engine_host.h @@ -187,7 +187,7 @@ public: }; void AnalyzeRowType(TStructLiteral* columnIds, TSmallVec<NTable::TTag>& tags, TSmallVec<NTable::TTag>& systemColumnTags); -NUdf::TUnboxedValue GetCellValue(const TCell& cell, NScheme::TTypeId type); +NUdf::TUnboxedValue GetCellValue(const TCell& cell, NScheme::TTypeInfo type); NUdf::TUnboxedValue CreateSelectRangeLazyRowsList(NTable::TDatabase& db, const NTable::TScheme& scheme, const THolderFactory& holderFactory, const TTableId& tableId, ui64 localTid, const TSmallVec<NTable::TTag>& tags, const TSmallVec<bool>& skipNullKeys, const TTableRange& range, ui64 itemsLimit, ui64 bytesLimit, diff --git a/ydb/core/engine/mkql_engine_flat.cpp b/ydb/core/engine/mkql_engine_flat.cpp index 52a61a3eb22..a79fdf83fe4 100644 --- a/ydb/core/engine/mkql_engine_flat.cpp +++ b/ydb/core/engine/mkql_engine_flat.cpp @@ -566,8 +566,8 @@ public: // Temporary check for KIKIMR-7112 bool CheckValidUint8InKey(TKeyDesc& desc) const { if (!desc.Range.Point) { - for (NScheme::TTypeId typeId : desc.KeyColumnTypes) { - if (typeId == NScheme::NTypeIds::Uint8) { + for (auto typeInfo : desc.KeyColumnTypes) { + if (typeInfo.GetTypeId() == NScheme::NTypeIds::Uint8) { AddError("Validate", __LINE__, "Bad shard program: dynamic keys with Uint8 columns are currently prohibited"); return false; } @@ -578,7 +578,7 @@ public: return false; } for (size_t i = 0; i < desc.Range.From.size(); ++i) { - if (desc.KeyColumnTypes[i] != NScheme::NTypeIds::Uint8) + if (desc.KeyColumnTypes[i].GetTypeId() != NScheme::NTypeIds::Uint8) continue; const TCell& c = desc.Range.From[i]; if (!c.IsNull() && c.AsValue<ui8>() > 127) { diff --git a/ydb/core/engine/mkql_engine_flat_extfunc.cpp b/ydb/core/engine/mkql_engine_flat_extfunc.cpp index 51e79ad54e4..bb10c3a8803 100644 --- a/ydb/core/engine/mkql_engine_flat_extfunc.cpp +++ b/ydb/core/engine/mkql_engine_flat_extfunc.cpp @@ -26,7 +26,9 @@ namespace { } const auto literal = AS_VALUE(TDataLiteral, data); - return MakeCell(literal->GetType()->GetSchemeType(), literal->AsValue(), env, false); + // TODO: support pg types + auto typeInfo = NScheme::TTypeInfo(literal->GetType()->GetSchemeType()); + return MakeCell(typeInfo, literal->AsValue(), env, false); } void ExtractRow(TVector<TCell>& row, TTupleLiteral* tupleNode, const TTypeEnvironment& env) { @@ -45,7 +47,9 @@ namespace { const auto dataType = type->IsOptional() ? AS_TYPE(TDataType, AS_TYPE(TOptionalType, type)->GetItemType()) : AS_TYPE(TDataType, type); - return MakeCell(dataType->GetSchemeType(), value, env); + // TODO: support pg types + auto typeInfo = NScheme::TTypeInfo(dataType->GetSchemeType()); + return MakeCell(typeInfo, value, env); } void ExtractRow(const NUdf::TUnboxedValue& row, TTupleType* rowType, TVector<TCell>& cells, const TTypeEnvironment& env) { @@ -600,7 +604,7 @@ namespace { bool truncatedAny = false; ui32 keyColumnsCount = 0; - TVector<NUdf::TDataTypeId> types; + TVector<NScheme::TTypeInfo> types; using TPartKey = std::tuple<const TCell*, NUdf::TUnboxedValue, ui64, ui64, NUdf::TUnboxedValue, bool>; @@ -632,17 +636,20 @@ namespace { ui32 typesSize = sizeof(ui32) + partKeyColumnsCount * sizeof(NUdf::TDataTypeId); MKQL_ENSURE(firstKey.size() >= typesSize, "Corrupted key"); const char* partTypes = firstKey.data() + sizeof(ui32); + if (!keyColumnsCount) { keyColumnsCount = partKeyColumnsCount; for (ui32 i = 0; i < partKeyColumnsCount; ++i) { auto partType = ReadUnaligned<NUdf::TDataTypeId>(partTypes + i * sizeof(NUdf::TDataTypeId)); - types.push_back(partType); + // TODO: support pg types + types.push_back(NScheme::TTypeInfo(partType)); } } else { MKQL_ENSURE(keyColumnsCount == partKeyColumnsCount, "Mismatch of key columns count"); for (ui32 i = 0; i < keyColumnsCount; ++i) { auto partType = ReadUnaligned<NUdf::TDataTypeId>(partTypes + i * sizeof(NUdf::TDataTypeId)); - MKQL_ENSURE(partType == types[i], "Mismatch of key columns type"); + // TODO: support pg types + MKQL_ENSURE(partType == types[i].GetTypeId(), "Mismatch of key columns type"); } } diff --git a/ydb/core/engine/mkql_engine_flat_ut.cpp b/ydb/core/engine/mkql_engine_flat_ut.cpp index 84fcf280cbe..145815e3c6e 100644 --- a/ydb/core/engine/mkql_engine_flat_ut.cpp +++ b/ydb/core/engine/mkql_engine_flat_ut.cpp @@ -15,6 +15,8 @@ namespace NKikimr { namespace NMiniKQL { namespace { + using TTypeInfo = NScheme::TTypeInfo; + const ui64 Shard1 = 100; const ui64 Shard2 = 101; //const ui64 Shard3 = 102; @@ -306,15 +308,15 @@ Y_UNIT_TEST_SUITE(TMiniKQLEngineFlatTest) { TRuntimeNode value = TRuntimeNode()) { auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)TKey; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(TKey); TRuntimeNode::TList row(1); row[0] = key ? key : pgmBuilder.NewNull(); auto update = pgmBuilder.GetUpdateRowBuilder(); ui32 columnId = (ui32)Schema1::Table1::Value::ColumnId; update.SetColumn(columnId, - TValue, + TTypeInfo(TValue), value ? value : pgmBuilder.NewNull()); auto pgm = pgmBuilder.Build(pgmBuilder.AsList(pgmBuilder.UpdateRow(TTableId(OwnerId, Table1Id), @@ -330,13 +332,13 @@ Y_UNIT_TEST_SUITE(TMiniKQLEngineFlatTest) { TDriver& driver) { auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)TKey; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(TKey); TRuntimeNode::TList row(1); row[0] = pgmBuilder.NewNull(); TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto value = pgmBuilder.SelectRow(TTableId(OwnerId, Table1Id), keyTypes, columns, row); @@ -420,8 +422,8 @@ Value { TDriver driver; driver.ShardDbState.AddShard<Schema1>(Shard1); auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList row(1); row[0] = pgmBuilder.TProgramBuilder::TProgramBuilder::NewDataLiteral<ui32>(42); TVector<TSelectColumn> columns; @@ -477,8 +479,8 @@ Value { } auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList row(1); row[0] = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(42); TVector<TSelectColumn> columns; @@ -536,12 +538,12 @@ Value { } auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList row(1); row[0] = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(42); TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto value = pgmBuilder.SelectRow(TTableId(OwnerId, Table1Id), keyTypes, columns, row); auto pgm = pgmBuilder.Build(pgmBuilder.AsList(pgmBuilder.SetResult("myRes", value))); @@ -685,8 +687,8 @@ Value { } auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList row(1); row[0] = pgmBuilder.Add( pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(40), @@ -727,8 +729,8 @@ Value { pgmBuilder.TProgramBuilder::NewDataLiteral<NUdf::EDataSlot::Utf8>("qwe")); } - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList row(1); row[0] = pgmBuilder.NewNull(); @@ -800,8 +802,8 @@ Value { driver.ShardDbState.AddShard<Schema1>(Shard1); auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList row(1); row[0] = pgmBuilder.Add( pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(40), @@ -838,8 +840,8 @@ Value { driver.ShardDbState.AddShard<Schema1>(Shard1); auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList row(1); row[0] = pgmBuilder.Add( pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(40), @@ -848,7 +850,7 @@ Value { auto update = pgmBuilder.GetUpdateRowBuilder(); ui32 columnId = (ui32)Schema1::Table1::Value::ColumnId; update.SetColumn(columnId, - NUdf::TDataType<NUdf::TUtf8>::Id, + TTypeInfo(NUdf::TDataType<NUdf::TUtf8>::Id), pgmBuilder.TProgramBuilder::NewDataLiteral<NUdf::EDataSlot::Utf8>("qwe")); auto pgm = pgmBuilder.Build(pgmBuilder.AsList(pgmBuilder.UpdateRow(TTableId(OwnerId, Table1Id), keyTypes, row, update))); @@ -910,8 +912,8 @@ Value { driver.ShardDbState.AddShard<Schema1>(Shard1); auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList row(1); row[0] = pgmBuilder.Add( pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(40), @@ -960,8 +962,8 @@ Value { } auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList row(1); row[0] = pgmBuilder.Add( pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(40), @@ -970,7 +972,7 @@ Value { auto update = pgmBuilder.GetUpdateRowBuilder(); ui32 columnId = (ui32)Schema1::Table1::Value::ColumnId; update.SetColumn(columnId, - NUdf::TDataType<NUdf::TUtf8>::Id, + TTypeInfo(NUdf::TDataType<NUdf::TUtf8>::Id), pgmBuilder.TProgramBuilder::NewDataLiteral<NUdf::EDataSlot::Utf8>("qwe")); auto pgm = pgmBuilder.Build(pgmBuilder.AsList(pgmBuilder.UpdateRow(TTableId(OwnerId, Table1Id), keyTypes, row, update))); @@ -1013,8 +1015,8 @@ Value { } auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList row(1); row[0] = pgmBuilder.Add( pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(40), @@ -1054,8 +1056,8 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -1125,8 +1127,8 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewNull(); options.FromColumns = rowFrom; @@ -1210,12 +1212,12 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -1329,11 +1331,11 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -1440,11 +1442,11 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewNull(); options.FromColumns = rowFrom; @@ -1554,12 +1556,12 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -1676,11 +1678,11 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(43); @@ -1798,11 +1800,11 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(43); @@ -1915,11 +1917,11 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); TRuntimeNode::TList rowTo(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); @@ -2040,11 +2042,11 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); TRuntimeNode::TList rowTo(1); rowFrom[0] = pgmBuilder.NewNull(); @@ -2151,11 +2153,11 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); TRuntimeNode::TList rowTo(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); @@ -2272,11 +2274,11 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); TRuntimeNode::TList rowTo(1); rowFrom[0] = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(42); @@ -2402,11 +2404,11 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); TRuntimeNode::TList rowTo(1); rowFrom[0] = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(42); @@ -2525,11 +2527,11 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); TRuntimeNode::TList rowTo(1); rowFrom[0] = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(42); @@ -2648,11 +2650,11 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); TRuntimeNode::TList rowTo(1); rowFrom[0] = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(42); @@ -2766,22 +2768,22 @@ Value { auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes1(1); - keyTypes1[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes1(1); + keyTypes1[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList row1(1); TVector<TSelectColumn> columns1; - columns1.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns1.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); row1[0] = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(42); auto value1 = pgmBuilder.SelectRow(TTableId(OwnerId, Table1Id), keyTypes1, columns1, row1); - TVector<ui32> keyTypes2(1); - keyTypes2[0] = (ui32)NUdf::TDataType<ui64>::Id; + TVector<TTypeInfo> keyTypes2(1); + keyTypes2[0] = TTypeInfo(NUdf::TDataType<ui64>::Id); TRuntimeNode::TList row2(1); row2[0] = pgmBuilder.TProgramBuilder::NewDataLiteral<ui64>(11); TVector<TSelectColumn> columns2; - columns2.emplace_back("4", (ui32)Schema2::Table2::Value::ColumnId, (ui32)Schema2::Table2::Value::ColumnType, + columns2.emplace_back("4", (ui32)Schema2::Table2::Value::ColumnId, TTypeInfo(Schema2::Table2::Value::ColumnType), EColumnTypeConstraint::Nullable); auto value2 = pgmBuilder.SelectRow(TTableId(OwnerId, Table2Id), keyTypes2, columns2, row2); @@ -2917,14 +2919,14 @@ Value { auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes1(1); - keyTypes1[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes1(1); + keyTypes1[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList row1(1); row1[0] = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(42); auto value1 = pgmBuilder.EraseRow(TTableId(OwnerId, Table1Id), keyTypes1, row1); - TVector<ui32> keyTypes2(1); - keyTypes2[0] = (ui32)NUdf::TDataType<ui64>::Id; + TVector<TTypeInfo> keyTypes2(1); + keyTypes2[0] = TTypeInfo(NUdf::TDataType<ui64>::Id); TRuntimeNode::TList row2(1); row2[0] = pgmBuilder.TProgramBuilder::NewDataLiteral<ui64>(11); @@ -2973,27 +2975,27 @@ Value { auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes1(1); - keyTypes1[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes1(1); + keyTypes1[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList row1(1); row1[0] = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(42); auto update1 = pgmBuilder.GetUpdateRowBuilder(); ui32 columnId1 = (ui32)Schema1::Table1::Value::ColumnId; update1.SetColumn(columnId1, - NUdf::TDataType<NUdf::TUtf8>::Id, + TTypeInfo(NUdf::TDataType<NUdf::TUtf8>::Id), pgmBuilder.TProgramBuilder::NewDataLiteral<NUdf::EDataSlot::Utf8>("qwe")); auto value1 = pgmBuilder.UpdateRow(TTableId(OwnerId, Table1Id), keyTypes1, row1, update1); - TVector<ui32> keyTypes2(1); - keyTypes2[0] = (ui32)NUdf::TDataType<ui64>::Id; + TVector<TTypeInfo> keyTypes2(1); + keyTypes2[0] = TTypeInfo(NUdf::TDataType<ui64>::Id); TRuntimeNode::TList row2(1); row2[0] = pgmBuilder.TProgramBuilder::NewDataLiteral<ui64>(11); auto update2 = pgmBuilder.GetUpdateRowBuilder(); ui32 columnId2 = (ui32)Schema2::Table2::Value::ColumnId; update2.SetColumn(columnId2, - NUdf::TDataType<ui32>::Id, + TTypeInfo(NUdf::TDataType<ui32>::Id), pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(21)); auto value2 = pgmBuilder.UpdateRow(TTableId(OwnerId, Table2Id), keyTypes2, row2, update2); @@ -3065,36 +3067,36 @@ Value { auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes1(1); - keyTypes1[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes1(1); + keyTypes1[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList row1(1); row1[0] = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(42); auto update1 = pgmBuilder.GetUpdateRowBuilder(); ui32 columnId1 = (ui32)Schema1::Table1::Value::ColumnId; update1.SetColumn(columnId1, - NUdf::TDataType<NUdf::TUtf8>::Id, + TTypeInfo(NUdf::TDataType<NUdf::TUtf8>::Id), pgmBuilder.TProgramBuilder::NewDataLiteral<NUdf::EDataSlot::Utf8>("asd")); TVector<TSelectColumn> columns1; - columns1.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns1.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto oldRow1 = pgmBuilder.SelectRow(TTableId(OwnerId, Table1Id), keyTypes1, columns1, row1); auto value1 = pgmBuilder.UpdateRow(TTableId(OwnerId, Table1Id), keyTypes1, row1, update1); - TVector<ui32> keyTypes2(1); - keyTypes2[0] = (ui32)NUdf::TDataType<ui64>::Id; + TVector<TTypeInfo> keyTypes2(1); + keyTypes2[0] = TTypeInfo(NUdf::TDataType<ui64>::Id); TRuntimeNode::TList row2(1); row2[0] = pgmBuilder.TProgramBuilder::NewDataLiteral<ui64>(11); auto update2 = pgmBuilder.GetUpdateRowBuilder(); ui32 columnId2 = (ui32)Schema2::Table2::Value::ColumnId; update2.SetColumn(columnId2, - NUdf::TDataType<ui32>::Id, + TTypeInfo(NUdf::TDataType<ui32>::Id), pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(32)); TVector<TSelectColumn> columns2; - columns2.emplace_back("4", (ui32)Schema2::Table2::Value::ColumnId, (ui32)Schema2::Table2::Value::ColumnType, + columns2.emplace_back("4", (ui32)Schema2::Table2::Value::ColumnId, TTypeInfo(Schema2::Table2::Value::ColumnType), EColumnTypeConstraint::Nullable); auto oldRow2 = pgmBuilder.SelectRow(TTableId(OwnerId, Table2Id), keyTypes2, columns2, row2); @@ -3189,12 +3191,12 @@ Value { } auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList row(1); row[0] = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(42); TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto value = pgmBuilder.SelectRow(TTableId(OwnerId, Table1Id), keyTypes, columns, row); auto pgm = pgmBuilder.Build(pgmBuilder.AsList(pgmBuilder.SetResult("myRes", value))); @@ -3268,12 +3270,12 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -3358,8 +3360,8 @@ Value { driver.ShardDbState.AddShard<Schema1>(Shard1); auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList row(1); row[0] = pgmBuilder.Add( pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(40), @@ -3368,7 +3370,7 @@ Value { auto update = pgmBuilder.GetUpdateRowBuilder(); ui32 columnId = (ui32)Schema1::Table1::Value::ColumnId; update.SetColumn(columnId, - NUdf::TDataType<NUdf::TUtf8>::Id, + TTypeInfo(NUdf::TDataType<NUdf::TUtf8>::Id), pgmBuilder.TProgramBuilder::NewDataLiteral<NUdf::EDataSlot::Utf8>("qwe")); auto pgm = pgmBuilder.Build(pgmBuilder.AsList(pgmBuilder.UpdateRow(TTableId(OwnerId, Table1Id), keyTypes, row, update))); @@ -3409,8 +3411,8 @@ Value { } auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList row(1); row[0] = pgmBuilder.Add( pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(40), @@ -3466,12 +3468,12 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -3592,12 +3594,12 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -3722,12 +3724,12 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -3856,12 +3858,12 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -4003,12 +4005,12 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -4130,12 +4132,12 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -4264,12 +4266,12 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("2", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -4767,14 +4769,14 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, + columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -4845,14 +4847,14 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, + columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -4915,21 +4917,21 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, + columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; options.Flags = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(TReadRangeOptions::TFlags::IncludeTermValue); auto range = pgmBuilder.SelectRange(TTableId(OwnerId, Table1Id), keyTypes, columns, options); auto list = pgmBuilder.Member(range, "List"); - auto mapped = pgmBuilder.Map(list, [&pgmBuilder, keyTypes, columns](TRuntimeNode item) { + auto mapped = pgmBuilder.Map(list, [&pgmBuilder, &keyTypes, &columns](TRuntimeNode item) { TRuntimeNode::TList row(1); row[0] = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(3); @@ -4979,14 +4981,14 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, + columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -5067,14 +5069,14 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, + columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -5087,15 +5089,15 @@ Value { }); auto write = pgmBuilder.Map(mapped, [&pgmBuilder](TRuntimeNode item) { - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui64>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui64>::Id); TRuntimeNode::TList row(1); row[0] = pgmBuilder.Convert(pgmBuilder.Member(item, "ID"), pgmBuilder.NewOptionalType(pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id))); auto update = pgmBuilder.GetUpdateRowBuilder(); ui32 columnId = (ui32)Schema2::Table2::Value::ColumnId; - update.SetColumn(columnId, NUdf::TDataType<ui32>::Id, pgmBuilder.Member(item, "ID")); + update.SetColumn(columnId, TTypeInfo(NUdf::TDataType<ui32>::Id), pgmBuilder.Member(item, "ID")); return pgmBuilder.UpdateRow(TTableId(OwnerId, Table2Id), keyTypes, row, update); }); @@ -5112,14 +5114,14 @@ Value { { TVector<TSelectColumn> columns; - columns.emplace_back("ID", (ui32)Schema2::Table2::ID::ColumnId, (ui32)Schema2::Table2::ID::ColumnType, + columns.emplace_back("ID", (ui32)Schema2::Table2::ID::ColumnId, TTypeInfo(Schema2::Table2::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns.emplace_back("Value", (ui32)Schema2::Table2::Value::ColumnId, (ui32)Schema2::Table2::Value::ColumnType, + columns.emplace_back("Value", (ui32)Schema2::Table2::Value::ColumnId, TTypeInfo(Schema2::Table2::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui64>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui64>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui64>::Id); options.FromColumns = rowFrom; @@ -5176,13 +5178,13 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns1; - columns1.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, + columns1.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns1.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns1.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options1 = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes1(1); - keyTypes1[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes1(1); + keyTypes1[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom1(1); rowFrom1[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options1.FromColumns = rowFrom1; @@ -5191,13 +5193,13 @@ Value { auto list1 = pgmBuilder.Member(range1, "List"); TVector<TSelectColumn> columns2; - columns2.emplace_back("ID", (ui32)Schema2::Table2::ID::ColumnId, (ui32)Schema2::Table2::ID::ColumnType, + columns2.emplace_back("ID", (ui32)Schema2::Table2::ID::ColumnId, TTypeInfo(Schema2::Table2::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns2.emplace_back("Value", (ui32)Schema2::Table2::Value::ColumnId, (ui32)Schema2::Table2::Value::ColumnType, + columns2.emplace_back("Value", (ui32)Schema2::Table2::Value::ColumnId, TTypeInfo(Schema2::Table2::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options2 = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes2(1); - keyTypes2[0] = (ui32)NUdf::TDataType<ui64>::Id; + TVector<TTypeInfo> keyTypes2(1); + keyTypes2[0] = TTypeInfo(NUdf::TDataType<ui64>::Id); TRuntimeNode::TList rowFrom2(1); rowFrom2[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui64>::Id); options2.FromColumns = rowFrom2; @@ -5270,13 +5272,13 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns1; - columns1.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, + columns1.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns1.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns1.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options1 = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes1(1); - keyTypes1[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes1(1); + keyTypes1[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom1(1); rowFrom1[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options1.FromColumns = rowFrom1; @@ -5285,13 +5287,13 @@ Value { auto list1 = pgmBuilder.Member(range1, "List"); TVector<TSelectColumn> columns2; - columns2.emplace_back("ID", (ui32)Schema2::Table2::ID::ColumnId, (ui32)Schema2::Table2::ID::ColumnType, + columns2.emplace_back("ID", (ui32)Schema2::Table2::ID::ColumnId, TTypeInfo(Schema2::Table2::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns2.emplace_back("Value", (ui32)Schema2::Table2::Value::ColumnId, (ui32)Schema2::Table2::Value::ColumnType, + columns2.emplace_back("Value", (ui32)Schema2::Table2::Value::ColumnId, TTypeInfo(Schema2::Table2::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options2 = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes2(1); - keyTypes2[0] = (ui32)NUdf::TDataType<ui64>::Id; + TVector<TTypeInfo> keyTypes2(1); + keyTypes2[0] = TTypeInfo(NUdf::TDataType<ui64>::Id); TRuntimeNode::TList rowFrom2(1); rowFrom2[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui64>::Id); options2.FromColumns = rowFrom2; @@ -5377,14 +5379,14 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, + columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -5493,14 +5495,14 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, + columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -5604,14 +5606,14 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, + columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -5688,14 +5690,14 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, + columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -5766,14 +5768,14 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, + columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -5847,14 +5849,14 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, + columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -5936,14 +5938,14 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, + columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -6023,14 +6025,14 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, + columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -6114,14 +6116,14 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, + columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -6209,14 +6211,14 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns; - columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, + columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable); - columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, + columns.emplace_back("Value", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable); auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -6284,8 +6286,8 @@ Value { auto& pgmBuilder = driver.PgmBuilder; auto options = pgmBuilder.GetDefaultTableRangeOptions(); - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TRuntimeNode::TList rowFrom(1); rowFrom[0] = pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); options.FromColumns = rowFrom; @@ -6360,16 +6362,17 @@ Value { auto& pgmBuilder = driver.PgmBuilder; TVector<TSelectColumn> columns1 { - {"ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, EColumnTypeConstraint::Nullable}, - {"Value", (ui32)Schema1::Table1::Value::ColumnId, (ui32)Schema1::Table1::Value::ColumnType, EColumnTypeConstraint::Nullable} + {"ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable}, + {"Value", (ui32)Schema1::Table1::Value::ColumnId, TTypeInfo(Schema1::Table1::Value::ColumnType), EColumnTypeConstraint::Nullable} }; TRuntimeNode::TList fromColumns1{pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id)}; auto options1 = pgmBuilder.GetDefaultTableRangeOptions(); options1.FromColumns = fromColumns1; options1.Flags = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(TReadRangeOptions::TFlags::IncludeTermValue); - auto range1 = pgmBuilder.SelectRange(TTableId(OwnerId, Table1Id), {(ui32)NUdf::TDataType<ui32>::Id}, - columns1, options1); + TVector<TTypeInfo> keyTypes1(1); + keyTypes1[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); + auto range1 = pgmBuilder.SelectRange(TTableId(OwnerId, Table1Id), keyTypes1, columns1, options1); auto list1 = pgmBuilder.Member(range1, "List"); const TString filterValue = "Value6"; @@ -6383,16 +6386,17 @@ Value { auto take1 = pgmBuilder.Take(filtered1, pgmBuilder.TProgramBuilder::NewDataLiteral<ui64>(2)); TVector<TSelectColumn> columns2 { - {"ID", (ui32)Schema2::Table2::ID::ColumnId, (ui32)Schema2::Table2::ID::ColumnType, EColumnTypeConstraint::Nullable}, - {"Value", (ui32)Schema2::Table2::Value::ColumnId, (ui32)Schema2::Table2::Value::ColumnType, EColumnTypeConstraint::Nullable} + {"ID", (ui32)Schema2::Table2::ID::ColumnId, TTypeInfo(Schema2::Table2::ID::ColumnType), EColumnTypeConstraint::Nullable}, + {"Value", (ui32)Schema2::Table2::Value::ColumnId, TTypeInfo(Schema2::Table2::Value::ColumnType), EColumnTypeConstraint::Nullable} }; TRuntimeNode::TList fromColumns2{pgmBuilder.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui64>::Id)}; auto options2 = pgmBuilder.GetDefaultTableRangeOptions(); options2.FromColumns = fromColumns2; options2.Flags = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(TReadRangeOptions::TFlags::IncludeTermValue); - auto range2 = pgmBuilder.SelectRange(TTableId(OwnerId, Table2Id), {(ui32)NUdf::TDataType<ui64>::Id}, - columns2, options2); + TVector<TTypeInfo> keyTypes2(1); + keyTypes2[0] = TTypeInfo(NUdf::TDataType<ui64>::Id); + auto range2 = pgmBuilder.SelectRange(TTableId(OwnerId, Table2Id), keyTypes2, columns2, options2); auto list2 = pgmBuilder.Member(range2, "List"); auto mapped2 = pgmBuilder.Map(list2, [&pgmBuilder](TRuntimeNode item) { @@ -6435,10 +6439,10 @@ Value { auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TVector<TSelectColumn> columns; - columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, + columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable); TRuntimeNode::TList row1(1); @@ -6484,14 +6488,14 @@ Value { auto& pgmBuilder = driver.PgmBuilder; - TVector<ui32> keyTypes(1); - keyTypes[0] = (ui32)NUdf::TDataType<ui32>::Id; + TVector<TTypeInfo> keyTypes(1); + keyTypes[0] = TTypeInfo(NUdf::TDataType<ui32>::Id); TVector<TSelectColumn> columns; - columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, (ui32)Schema1::Table1::ID::ColumnType, + columns.emplace_back("ID", (ui32)Schema1::Table1::ID::ColumnId, TTypeInfo(Schema1::Table1::ID::ColumnType), EColumnTypeConstraint::Nullable); - TVector<ui32> key2Types(1); - key2Types[0] = (ui32)NUdf::TDataType<ui64>::Id; + TVector<TTypeInfo> key2Types(1); + key2Types[0] = TTypeInfo(NUdf::TDataType<ui64>::Id); TRuntimeNode::TList row1(1); row1[0] = pgmBuilder.TProgramBuilder::NewDataLiteral<ui32>(1); diff --git a/ydb/core/engine/mkql_keys.cpp b/ydb/core/engine/mkql_keys.cpp index 720e00325e7..fb34be11586 100644 --- a/ydb/core/engine/mkql_keys.cpp +++ b/ydb/core/engine/mkql_keys.cpp @@ -48,7 +48,7 @@ TCell MakeCell(const NUdf::TUnboxedValuePod& value) { THolder<TKeyDesc> ExtractKeyTuple(const TTableId& tableId, TTupleLiteral* tuple, const TVector<TKeyDesc::TColumnOp>& columns, TKeyDesc::ERowOperation rowOperation, bool requireStaticKey, const TTypeEnvironment& env) { - TVector<ui32> keyColumnTypes(tuple->GetValuesCount()); + TVector<NScheme::TTypeInfo> keyColumnTypes(tuple->GetValuesCount()); TVector<TCell> fromValues(tuple->GetValuesCount()); TVector<TCell> toValues(tuple->GetValuesCount()); bool inclusiveFrom = true; @@ -59,7 +59,8 @@ THolder<TKeyDesc> ExtractKeyTuple(const TTableId& tableId, TTupleLiteral* tuple, auto type = tuple->GetType()->GetElementType(i); bool isOptional; auto dataType = UnpackOptionalData(type, isOptional); - keyColumnTypes[i] = dataType->GetSchemeType(); + // TODO: support pg types + keyColumnTypes[i] = NScheme::TTypeInfo(dataType->GetSchemeType()); if (i != staticComponents) { continue; } @@ -74,6 +75,8 @@ THolder<TKeyDesc> ExtractKeyTuple(const TTableId& tableId, TTupleLiteral* tuple, } ++staticComponents; + // TODO: support pg types + Y_VERIFY(keyColumnTypes[i].GetTypeId() != NScheme::NTypeIds::Pg); fromValues[i] = toValues[i] = MakeCell(keyColumnTypes[i], data, env); } @@ -92,8 +95,11 @@ void ExtractReadColumns(TStructType* columnsType, TStructLiteral* tags, TVector< op.Column = columnId; op.Operation = TKeyDesc::EColumnOperation::Read; bool isOptional; - op.ExpectedType = UnpackOptionalData(columnsType->GetMemberType(i), isOptional)->GetSchemeType(); - MKQL_ENSURE(op.ExpectedType != 0, "Null type is not allowed"); + auto dataType = UnpackOptionalData(columnsType->GetMemberType(i), isOptional); + auto expectedType = (NScheme::TTypeId)dataType->GetSchemeType(); + MKQL_ENSURE(expectedType != 0, "Null type is not allowed"); + // TODO: support pg types + op.ExpectedType = NScheme::TTypeInfo(expectedType); } } @@ -130,7 +136,7 @@ THolder<TKeyDesc> ExtractSelectRange(TCallable& callable, const TTypeEnvironment ui64 itemsLimit = AS_VALUE(TDataLiteral, callable.GetInput(6))->AsValue().Get<ui64>(); ui64 bytesLimit = AS_VALUE(TDataLiteral, callable.GetInput(7))->AsValue().Get<ui64>(); - TVector<ui32> keyColumnTypes(Max(fromTuple->GetValuesCount(), toTuple->GetValuesCount())); + TVector<NScheme::TTypeInfo> keyColumnTypes(Max(fromTuple->GetValuesCount(), toTuple->GetValuesCount())); TVector<TCell> fromValues(keyColumnTypes.size()); // padded with NULLs TVector<TCell> toValues(toTuple->GetValuesCount()); bool inclusiveFrom = !(flags & TReadRangeOptions::TFlags::ExcludeInitValue); @@ -140,7 +146,9 @@ THolder<TKeyDesc> ExtractSelectRange(TCallable& callable, const TTypeEnvironment auto type = fromTuple->GetType()->GetElementType(i); bool isOptional; auto dataType = UnpackOptionalData(type, isOptional); - keyColumnTypes[i] = dataType->GetSchemeType(); + auto keyType = (NScheme::TTypeId)dataType->GetSchemeType(); + // TODO: support pg types + keyColumnTypes[i] = NScheme::TTypeInfo(keyType); auto valueNode = fromTuple->GetValue(i); NUdf::TUnboxedValue data; bool hasImmediateData = ExtractKeyData(valueNode, isOptional, data); @@ -152,7 +160,9 @@ THolder<TKeyDesc> ExtractSelectRange(TCallable& callable, const TTypeEnvironment auto type = toTuple->GetType()->GetElementType(i); bool isOptional; auto dataType = UnpackOptionalData(type, isOptional); - keyColumnTypes[i] = dataType->GetSchemeType(); + auto keyType = (NScheme::TTypeId)dataType->GetSchemeType(); + // TODO: support pg types + keyColumnTypes[i] = NScheme::TTypeInfo(keyType); auto valueNode = toTuple->GetValue(i); NUdf::TUnboxedValue data; bool hasImmediateData = ExtractKeyData(valueNode, isOptional, data); @@ -191,7 +201,7 @@ THolder<TKeyDesc> ExtractUpdateRow(TCallable& callable, const TTypeEnvironment& if (cmd.GetStaticType()->IsVoid()) { // erase op.Operation = TKeyDesc::EColumnOperation::Set; - op.ExpectedType = 0; + op.ExpectedType = NScheme::TTypeInfo(0); } else if (cmd.GetStaticType()->IsTuple()) { // inplace update TTupleLiteral* tuple = AS_VALUE(TTupleLiteral, cmd); @@ -203,7 +213,9 @@ THolder<TKeyDesc> ExtractUpdateRow(TCallable& callable, const TTypeEnvironment& auto valueNode = tuple->GetValue(1); op.Operation = TKeyDesc::EColumnOperation::InplaceUpdate; bool isOptional; - op.ExpectedType = UnpackOptionalData(valueNode, isOptional)->GetSchemeType(); + auto keyType = (NScheme::TTypeId)UnpackOptionalData(valueNode, isOptional)->GetSchemeType(); + // TODO: support pg types + op.ExpectedType = NScheme::TTypeInfo(keyType); MKQL_ENSURE(!isOptional, "Expected data type for inplace update, not an optional"); op.InplaceUpdateMode = mode; @@ -214,10 +226,12 @@ THolder<TKeyDesc> ExtractUpdateRow(TCallable& callable, const TTypeEnvironment& } else { // update - op.Operation = TKeyDesc::EColumnOperation::Set; - bool isOptional; - op.ExpectedType = UnpackOptionalData(cmd, isOptional)->GetSchemeType(); - MKQL_ENSURE(op.ExpectedType != 0, "Null type is not allowed"); + op.Operation = TKeyDesc::EColumnOperation::Set; + bool isOptional; + auto keyType = (NScheme::TTypeId)UnpackOptionalData(cmd, isOptional)->GetSchemeType(); + // TODO: support pg types + op.ExpectedType = NScheme::TTypeInfo(keyType); + MKQL_ENSURE(op.ExpectedType.GetTypeId() != 0, "Null type is not allowed"); NUdf::TUnboxedValue data; if (ExtractKeyData(cmd, isOptional, data)) { @@ -243,11 +257,14 @@ THolder<TKeyDesc> ExtractEraseRow(TCallable& callable, const TTypeEnvironment& e #define MAKE_PRIMITIVE_TYPE_CELL(type, layout) \ case NUdf::TDataType<type>::Id: return MakeCell<layout>(value); -TCell MakeCell(NUdf::TDataTypeId typeId, const NUdf::TUnboxedValuePod& value, const TTypeEnvironment& env, bool copy) { +TCell MakeCell(NScheme::TTypeInfo type, const NUdf::TUnboxedValuePod& value, const TTypeEnvironment& env, bool copy) { if (!value) return TCell(); - switch(typeId) { + // TODO: support pg types + Y_VERIFY(type.GetTypeId() != NScheme::NTypeIds::Pg, "pg types are not supported"); + + switch(type.GetTypeId()) { KNOWN_FIXED_VALUE_TYPES(MAKE_PRIMITIVE_TYPE_CELL) case NUdf::TDataType<NUdf::TDecimal>::Id: { @@ -336,7 +353,7 @@ TTableId ExtractTableId(const TRuntimeNode& node) { } void FillKeyTupleValue(const NUdf::TUnboxedValue& row, const TVector<ui32>& rowIndices, - const TVector<NUdf::TDataTypeId>& rowTypes, TVector<TCell>& cells, const TTypeEnvironment& env) + const TVector<NScheme::TTypeInfo>& rowTypes, TVector<TCell>& cells, const TTypeEnvironment& env) { for (ui32 i = 0; i < rowIndices.size(); ++i) { auto rowIndex = rowIndices[i]; diff --git a/ydb/core/engine/mkql_keys.h b/ydb/core/engine/mkql_keys.h index 635ad3ce147..ac968a473e9 100644 --- a/ydb/core/engine/mkql_keys.h +++ b/ydb/core/engine/mkql_keys.h @@ -44,9 +44,9 @@ struct TTableStrings { THolder<TKeyDesc> ExtractTableKey(TCallable& callable, const TTableStrings& strings, const TTypeEnvironment& env); TVector<THolder<TKeyDesc>> ExtractTableKeys(TExploringNodeVisitor& explorer, const TTypeEnvironment& env); TTableId ExtractTableId(const TRuntimeNode& node); -TCell MakeCell(NUdf::TDataTypeId typeId, const NUdf::TUnboxedValuePod& value, const TTypeEnvironment& env, bool copy = true); +TCell MakeCell(NScheme::TTypeInfo type, const NUdf::TUnboxedValuePod& value, const TTypeEnvironment& env, bool copy = true); void FillKeyTupleValue(const NUdf::TUnboxedValue& row, const TVector<ui32>& rowIndices, - const TVector<NUdf::TDataTypeId>& rowTypes, TVector<TCell>& cells, const TTypeEnvironment& env); + const TVector<NScheme::TTypeInfo>& rowTypes, TVector<TCell>& cells, const TTypeEnvironment& env); } } diff --git a/ydb/core/engine/mkql_proto.cpp b/ydb/core/engine/mkql_proto.cpp index ea0ac631994..6fcb2de9fb8 100644 --- a/ydb/core/engine/mkql_proto.cpp +++ b/ydb/core/engine/mkql_proto.cpp @@ -156,36 +156,19 @@ NUdf::TUnboxedValue ImportValueFromProto(TType* type, const Ydb::Value& value, c return std::move(unboxedValue); } + case TType::EKind::Pg: + // TODO: support pg types + MKQL_ENSURE(false, "pg types are not supported"); + default: MKQL_ENSURE(false, TStringBuilder() << "Unknown kind: " << type->GetKindAsStr()); } } - -template <typename ValType> -class TAlmostDoneTypeValue : public TRawTypeValue { -public: - TAlmostDoneTypeValue(NUdf::TDataTypeId schemeType, ValType value) - : TRawTypeValue(&Value, sizeof(Value), schemeType) - , Value(value) - {} - -protected: - ValType Value; -}; - -template <> -class TAlmostDoneTypeValue<TString> : public TRawTypeValue { -public: - TAlmostDoneTypeValue(NUdf::TDataTypeId schemeType, const TString& value) - : TRawTypeValue(value.data(), value.size(), schemeType) - {} -}; - // NOTE: TCell's can reference memomry from tupleValue bool CellsFromTuple(const NKikimrMiniKQL::TType* tupleType, const NKikimrMiniKQL::TValue& tupleValue, - const TConstArrayRef<NScheme::TTypeId>& types, + const TConstArrayRef<NScheme::TTypeInfo>& types, bool allowCastFromString, TVector<TCell>& key, TString& errStr) @@ -209,9 +192,9 @@ bool CellsFromTuple(const NKikimrMiniKQL::TType* tupleType, const auto& item = ti.GetOptional().GetItem(); CHECK_OR_RETURN_ERROR(item.GetKind() == NKikimrMiniKQL::Data, "Element at index " + ToString(i) + " Item kind is not Data"); const auto& typeId = item.GetData().GetScheme(); - CHECK_OR_RETURN_ERROR(typeId == types[i] || + CHECK_OR_RETURN_ERROR(typeId == types[i].GetTypeId() || allowCastFromString && (typeId == NScheme::NTypeIds::Utf8), - "Element at index " + ToString(i) + " has type " + ToString(typeId) + " but expected type is " + ToString(types[i])); + "Element at index " + ToString(i) + " has type " + ToString(typeId) + " but expected type is " + ToString(types[i].GetTypeId())); } CHECK_OR_RETURN_ERROR(tupleType->GetTuple().ElementSize() == tupleValue.TupleSize(), @@ -256,7 +239,8 @@ bool CellsFromTuple(const NKikimrMiniKQL::TType* tupleType, Sprintf("Simple type is expected in tuple at position %" PRIu32, i)); TCell c; - switch (types[i]) { + auto typeId = types[i].GetTypeId(); + switch (typeId) { #define CASE_SIMPLE_TYPE(name, type, protoField) \ case NScheme::NTypeIds::name: \ @@ -266,7 +250,7 @@ bool CellsFromTuple(const NKikimrMiniKQL::TType* tupleType, type val = v.Get##protoField(); \ c = TCell((const char*)&val, sizeof(val)); \ } else if (allowCastFromString && v.HasText()) { \ - const auto slot = NUdf::GetDataSlot(types[i]); \ + const auto slot = NUdf::GetDataSlot(typeId); \ const auto out = NMiniKQL::ValueFromString(slot, v.GetText()); \ CHECK_OR_RETURN_ERROR(out, Sprintf("Cannot parse value of type " #name " from text '%s' in tuple at position %" PRIu32, v.GetText().data(), i)); \ const auto val = out.Get<type>(); \ @@ -321,8 +305,13 @@ bool CellsFromTuple(const NKikimrMiniKQL::TType* tupleType, } break; } + case NScheme::NTypeIds::Pg: + // TODO: support pg types + CHECK_OR_RETURN_ERROR(false, Sprintf("Unsupported pg type at position %" PRIu32, i)); + break; + default: - CHECK_OR_RETURN_ERROR(false, Sprintf("Unsupported typeId %" PRIu16 " at index %" PRIu32, types[i], i)); + CHECK_OR_RETURN_ERROR(false, Sprintf("Unsupported typeId %" PRIu16 " at index %" PRIu32, typeId, i)); break; } @@ -335,11 +324,12 @@ bool CellsFromTuple(const NKikimrMiniKQL::TType* tupleType, return true; } -bool CellToValue(NScheme::TTypeId typeId, const TCell& c, NKikimrMiniKQL::TValue& val, TString& errStr) { +bool CellToValue(NScheme::TTypeInfo type, const TCell& c, NKikimrMiniKQL::TValue& val, TString& errStr) { if (c.IsNull()) { return true; } + auto typeId = type.GetTypeId(); switch (typeId) { case NScheme::NTypeIds::Int8: Y_VERIFY(c.Size() == sizeof(i8)); @@ -420,6 +410,12 @@ bool CellToValue(NScheme::TTypeId typeId, const TCell& c, NKikimrMiniKQL::TValue case NScheme::NTypeIds::Utf8: val.MutableOptional()->SetText(c.Data(), c.Size()); break; + + case NScheme::NTypeIds::Pg: + // TODO: support pg types + errStr = "Unknown pg type"; + return false; + default: errStr = "Unknown type: " + ToString(typeId); return false; diff --git a/ydb/core/engine/mkql_proto.h b/ydb/core/engine/mkql_proto.h index 510ff15991f..06d1bbce15c 100644 --- a/ydb/core/engine/mkql_proto.h +++ b/ydb/core/engine/mkql_proto.h @@ -17,12 +17,12 @@ NUdf::TUnboxedValue ImportValueFromProto(TType* type, const Ydb::Value& value, c // NOTE: TCell's can reference memomry from tupleValue bool CellsFromTuple(const NKikimrMiniKQL::TType* tupleType, const NKikimrMiniKQL::TValue& tupleValue, - const TConstArrayRef<NScheme::TTypeId>& expectedTypes, + const TConstArrayRef<NScheme::TTypeInfo>& expectedTypes, bool allowCastFromString, TVector<TCell>& key, TString& errStr); -bool CellToValue(NScheme::TTypeId typeId, const TCell& c, NKikimrMiniKQL::TValue& val, TString& errStr); +bool CellToValue(NScheme::TTypeInfo type, const TCell& c, NKikimrMiniKQL::TValue& val, TString& errStr); } // namspace NMiniKQL } // namspace NKikimr diff --git a/ydb/core/engine/mkql_proto_ut.cpp b/ydb/core/engine/mkql_proto_ut.cpp index c6e00e947c5..5ac30073b5d 100644 --- a/ydb/core/engine/mkql_proto_ut.cpp +++ b/ydb/core/engine/mkql_proto_ut.cpp @@ -389,7 +389,7 @@ Y_UNIT_TEST(TestExportVariantStructTypeYdb) { "variant_index: 1\n"); } - TString DoTestCellsFromTuple(const TConstArrayRef<NScheme::TTypeId>& types, TString paramsProto) { + TString DoTestCellsFromTuple(const TConstArrayRef<NScheme::TTypeInfo>& types, TString paramsProto) { NKikimrMiniKQL::TParams params; bool parseOk = ::google::protobuf::TextFormat::ParseFromString(paramsProto, ¶ms); UNIT_ASSERT_C(parseOk, paramsProto); @@ -405,7 +405,7 @@ Y_UNIT_TEST(TestExportVariantStructTypeYdb) { Y_UNIT_TEST(TestCellsFromTuple) { UNIT_ASSERT_VALUES_EQUAL("", DoTestCellsFromTuple( { - NScheme::NTypeIds::Int32 + NScheme::TTypeInfo(NScheme::NTypeIds::Int32) }, " Type {" " Kind : Tuple" @@ -421,7 +421,7 @@ Y_UNIT_TEST(TestExportVariantStructTypeYdb) { UNIT_ASSERT_VALUES_EQUAL("Value of type Int32 expected in tuple at position 0", DoTestCellsFromTuple( { - NScheme::NTypeIds::Int32 + NScheme::TTypeInfo(NScheme::NTypeIds::Int32) }, " Type {" " Kind : Tuple" @@ -437,7 +437,7 @@ Y_UNIT_TEST(TestExportVariantStructTypeYdb) { UNIT_ASSERT_VALUES_EQUAL("", DoTestCellsFromTuple( { - NScheme::NTypeIds::Int32 + NScheme::TTypeInfo(NScheme::NTypeIds::Int32) }, " Type {" " Kind : Tuple" @@ -453,7 +453,7 @@ Y_UNIT_TEST(TestExportVariantStructTypeYdb) { UNIT_ASSERT_VALUES_EQUAL("", DoTestCellsFromTuple( { - NScheme::NTypeIds::Int32 + NScheme::TTypeInfo(NScheme::NTypeIds::Int32) }, " Type {" " Kind : Tuple" @@ -469,7 +469,7 @@ Y_UNIT_TEST(TestExportVariantStructTypeYdb) { UNIT_ASSERT_VALUES_EQUAL("", DoTestCellsFromTuple( { - NScheme::NTypeIds::String + NScheme::TTypeInfo(NScheme::NTypeIds::String) }, " Type {" " Kind : Tuple" @@ -485,7 +485,7 @@ Y_UNIT_TEST(TestExportVariantStructTypeYdb) { UNIT_ASSERT_VALUES_EQUAL("Cannot parse value of type Uint32 from text '-42' in tuple at position 0", DoTestCellsFromTuple( { - NScheme::NTypeIds::Uint32 + NScheme::TTypeInfo(NScheme::NTypeIds::Uint32) }, " Type {" " Kind : Tuple" @@ -501,7 +501,7 @@ Y_UNIT_TEST(TestExportVariantStructTypeYdb) { UNIT_ASSERT_VALUES_EQUAL("Tuple value length 0 doesn't match the length in type 1", DoTestCellsFromTuple( { - NScheme::NTypeIds::Int32 + NScheme::TTypeInfo(NScheme::NTypeIds::Int32) }, " Type {" " Kind : Tuple" @@ -516,7 +516,7 @@ Y_UNIT_TEST(TestExportVariantStructTypeYdb) { UNIT_ASSERT_VALUES_EQUAL("", DoTestCellsFromTuple( { - NScheme::NTypeIds::Int32 + NScheme::TTypeInfo(NScheme::NTypeIds::Int32) }, " Type {" " Kind : Tuple" @@ -530,7 +530,7 @@ Y_UNIT_TEST(TestExportVariantStructTypeYdb) { UNIT_ASSERT_VALUES_EQUAL("Data must be present at position 0", DoTestCellsFromTuple( { - NScheme::NTypeIds::Int32 + NScheme::TTypeInfo(NScheme::NTypeIds::Int32) }, " Type {" " Kind : Tuple" @@ -546,8 +546,8 @@ Y_UNIT_TEST(TestExportVariantStructTypeYdb) { UNIT_ASSERT_VALUES_EQUAL("Tuple value length 0 doesn't match the length in type 1", DoTestCellsFromTuple( { - NScheme::NTypeIds::Int32, - NScheme::NTypeIds::Utf8 + NScheme::TTypeInfo(NScheme::NTypeIds::Int32), + NScheme::TTypeInfo(NScheme::NTypeIds::Utf8) }, " Type {" " Kind : Tuple" @@ -562,8 +562,8 @@ Y_UNIT_TEST(TestExportVariantStructTypeYdb) { UNIT_ASSERT_VALUES_EQUAL("", DoTestCellsFromTuple( { - NScheme::NTypeIds::Int32, - NScheme::NTypeIds::Int32 + NScheme::TTypeInfo(NScheme::NTypeIds::Int32), + NScheme::TTypeInfo(NScheme::NTypeIds::Int32) }, " Type {" " Kind : Tuple" @@ -581,7 +581,7 @@ Y_UNIT_TEST(TestExportVariantStructTypeYdb) { UNIT_ASSERT_VALUES_EQUAL("Tuple size 2 is greater that expected size 1", DoTestCellsFromTuple( { - NScheme::NTypeIds::Int32 + NScheme::TTypeInfo(NScheme::NTypeIds::Int32) }, " Type {" " Kind : Tuple" diff --git a/ydb/core/engine/ut/CMakeLists.darwin.txt b/ydb/core/engine/ut/CMakeLists.darwin.txt index 94c41469d9b..1084b3c9bb8 100644 --- a/ydb/core/engine/ut/CMakeLists.darwin.txt +++ b/ydb/core/engine/ut/CMakeLists.darwin.txt @@ -26,6 +26,7 @@ target_link_libraries(ydb-core-engine-ut PUBLIC test-libs-table mkql_proto-ut-helpers udf-service-exception_policy + yql-sql-pg_dummy ) target_link_options(ydb-core-engine-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/engine/ut/CMakeLists.linux.txt b/ydb/core/engine/ut/CMakeLists.linux.txt index e6ce7faed25..ef05f08345d 100644 --- a/ydb/core/engine/ut/CMakeLists.linux.txt +++ b/ydb/core/engine/ut/CMakeLists.linux.txt @@ -26,6 +26,7 @@ target_link_libraries(ydb-core-engine-ut PUBLIC test-libs-table mkql_proto-ut-helpers udf-service-exception_policy + yql-sql-pg_dummy ) target_link_options(ydb-core-engine-ut PRIVATE -ldl diff --git a/ydb/core/formats/arrow_batch_builder.cpp b/ydb/core/formats/arrow_batch_builder.cpp index ee74796711a..b7107e7ce11 100644 --- a/ydb/core/formats/arrow_batch_builder.cpp +++ b/ydb/core/formats/arrow_batch_builder.cpp @@ -52,7 +52,7 @@ arrow::Status AppendCell(arrow::RecordBatchBuilder& builder, const TCell& cell, return AppendCell(*builder.GetFieldAs<TBuilderType>(colNum), cell); } -arrow::Status AppendCell(arrow::RecordBatchBuilder& builder, const TCell& cell, ui32 colNum, NScheme::TTypeId type) { +arrow::Status AppendCell(arrow::RecordBatchBuilder& builder, const TCell& cell, ui32 colNum, NScheme::TTypeInfo type) { arrow::Status result; auto callback = [&]<typename TType>(TTypeWrapper<TType> typeHolder) { Y_UNUSED(typeHolder); @@ -196,7 +196,7 @@ TArrowBatchBuilder::TArrowBatchBuilder(arrow::Compression::type codec) WriteOptions.use_threads = false; } -bool TArrowBatchBuilder::Start(const TVector<std::pair<TString, NScheme::TTypeId>>& ydbColumns) { +bool TArrowBatchBuilder::Start(const TVector<std::pair<TString, NScheme::TTypeInfo>>& ydbColumns) { YdbSchema = ydbColumns; auto schema = MakeArrowSchema(ydbColumns); auto status = arrow::RecordBatchBuilder::Make(schema, arrow::default_memory_pool(), RowsToReserve, &BatchBuilder); @@ -206,7 +206,7 @@ bool TArrowBatchBuilder::Start(const TVector<std::pair<TString, NScheme::TTypeId void TArrowBatchBuilder::AppendCell(const TCell& cell, ui32 colNum) { NumBytes += cell.Size(); - const ui32 ydbType = YdbSchema[colNum].second; + auto ydbType = YdbSchema[colNum].second; auto status = NKikimr::NArrow::AppendCell(*BatchBuilder, cell, colNum, ydbType); Y_VERIFY(status.ok()); } @@ -216,7 +216,7 @@ void TArrowBatchBuilder::AddRow(const TDbTupleRef& key, const TDbTupleRef& value auto fnAppendTuple = [&] (const TDbTupleRef& tuple, size_t offsetInRow) { for (size_t i = 0; i < tuple.ColumnCount; ++i) { - const ui32 ydbType = tuple.Types[i]; + auto ydbType = tuple.Types[i]; const ui32 colNum = offsetInRow + i; Y_VERIFY(ydbType == YdbSchema[colNum].second); auto& cell = tuple.Columns[i]; diff --git a/ydb/core/formats/arrow_batch_builder.h b/ydb/core/formats/arrow_batch_builder.h index ae15d175445..303e57d23c9 100644 --- a/ydb/core/formats/arrow_batch_builder.h +++ b/ydb/core/formats/arrow_batch_builder.h @@ -151,7 +151,7 @@ public: TArrowBatchBuilder(arrow::Compression::type codec = arrow::Compression::UNCOMPRESSED); ~TArrowBatchBuilder() = default; - bool Start(const TVector<std::pair<TString, NScheme::TTypeId>>& columns, + bool Start(const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns, ui64 maxRowsInBlock, ui64 maxBytesInBlock, TString& err) override { Y_UNUSED(maxRowsInBlock); Y_UNUSED(maxBytesInBlock); @@ -174,20 +174,20 @@ public: return NumBytes; } - bool Start(const TVector<std::pair<TString, NScheme::TTypeId>>& columns); + bool Start(const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns); std::shared_ptr<arrow::RecordBatch> FlushBatch(bool reinitialize); std::shared_ptr<arrow::RecordBatch> GetBatch() const { return Batch; } protected: void AppendCell(const TCell& cell, ui32 colNum); - const TVector<std::pair<TString, NScheme::TTypeId>>& GetYdbSchema() const { + const TVector<std::pair<TString, NScheme::TTypeInfo>>& GetYdbSchema() const { return YdbSchema; } private: arrow::ipc::IpcWriteOptions WriteOptions; - TVector<std::pair<TString, NScheme::TTypeId>> YdbSchema; + TVector<std::pair<TString, NScheme::TTypeInfo>> YdbSchema; std::unique_ptr<arrow::RecordBatchBuilder> BatchBuilder; std::shared_ptr<arrow::RecordBatch> Batch; size_t RowsToReserve{DEFAULT_ROWS_TO_RESERVE}; diff --git a/ydb/core/formats/arrow_helpers.cpp b/ydb/core/formats/arrow_helpers.cpp index 388c460413b..00feaabbf5e 100644 --- a/ydb/core/formats/arrow_helpers.cpp +++ b/ydb/core/formats/arrow_helpers.cpp @@ -205,7 +205,7 @@ std::shared_ptr<arrow::DataType> CreateEmptyArrowImpl<arrow::DurationType>() { return arrow::duration(arrow::TimeUnit::TimeUnit::MICRO); } -std::shared_ptr<arrow::DataType> GetArrowType(NScheme::TTypeId typeId) { +std::shared_ptr<arrow::DataType> GetArrowType(NScheme::TTypeInfo typeId) { std::shared_ptr<arrow::DataType> result; bool success = SwitchYqlTypeToArrowType(typeId, [&]<typename TType>(TTypeWrapper<TType> typeHolder) { Y_UNUSED(typeHolder); @@ -218,7 +218,7 @@ std::shared_ptr<arrow::DataType> GetArrowType(NScheme::TTypeId typeId) { return std::make_shared<arrow::NullType>(); } -std::vector<std::shared_ptr<arrow::Field>> MakeArrowFields(const TVector<std::pair<TString, NScheme::TTypeId>>& columns) { +std::vector<std::shared_ptr<arrow::Field>> MakeArrowFields(const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns) { std::vector<std::shared_ptr<arrow::Field>> fields; fields.reserve(columns.size()); for (auto& [name, ydbType] : columns) { @@ -228,7 +228,7 @@ std::vector<std::shared_ptr<arrow::Field>> MakeArrowFields(const TVector<std::pa return fields; } -std::shared_ptr<arrow::Schema> MakeArrowSchema(const TVector<std::pair<TString, NScheme::TTypeId>>& ydbColumns) { +std::shared_ptr<arrow::Schema> MakeArrowSchema(const TVector<std::pair<TString, NScheme::TTypeInfo>>& ydbColumns) { return std::make_shared<arrow::Schema>(MakeArrowFields(ydbColumns)); } @@ -953,8 +953,8 @@ std::shared_ptr<arrow::RecordBatch> SortBatch(const std::shared_ptr<arrow::Recor return Reorder(batch, sortPermutation); } -static bool ConvertData(TCell& cell, const NScheme::TTypeId& colType, TMemoryPool& memPool, TString& errorMessage) { - switch (colType) { +static bool ConvertData(TCell& cell, const NScheme::TTypeInfo& colType, TMemoryPool& memPool, TString& errorMessage) { + switch (colType.GetTypeId()) { case NScheme::NTypeIds::DyNumber: { const auto dyNumber = NDyNumber::ParseDyNumberString(cell.AsBuf()); if (!dyNumber.Defined()) { @@ -985,8 +985,8 @@ static bool ConvertData(TCell& cell, const NScheme::TTypeId& colType, TMemoryPoo } static std::shared_ptr<arrow::Array> ConvertColumn(const std::shared_ptr<arrow::Array>& column, - NScheme::TTypeId colType) { - if (colType == NScheme::NTypeIds::Decimal) { + NScheme::TTypeInfo colType) { + if (colType.GetTypeId() == NScheme::NTypeIds::Decimal) { return {}; } @@ -999,7 +999,7 @@ static std::shared_ptr<arrow::Array> ConvertColumn(const std::shared_ptr<arrow:: builder.Reserve(binaryArray.length()).ok(); // TODO: ReserveData - switch (colType) { + switch (colType.GetTypeId()) { case NScheme::NTypeIds::DyNumber: { for (i32 i = 0; i < binaryArray.length(); ++i) { auto value = binaryArray.Value(i); @@ -1030,7 +1030,7 @@ static std::shared_ptr<arrow::Array> ConvertColumn(const std::shared_ptr<arrow:: } std::shared_ptr<arrow::RecordBatch> ConvertColumns(const std::shared_ptr<arrow::RecordBatch>& batch, - const THashMap<TString, NScheme::TTypeId>& columnsToConvert) + const THashMap<TString, NScheme::TTypeInfo>& columnsToConvert) { std::vector<std::shared_ptr<arrow::Array>> columns = batch->columns(); for (i32 i = 0; i < batch->num_columns(); ++i) { @@ -1076,6 +1076,9 @@ bool TArrowToYdbConverter::Process(const arrow::RecordBatch& batch, TString& err for (; row < rowsUnroll; row += unroll) { ui32 col = 0; for (auto& [colName, colType] : YdbSchema) { + // TODO: support pg types + Y_VERIFY(colType.GetTypeId() != NScheme::NTypeIds::Pg, "pg types are not supported"); + auto& column = allColumns[col]; bool success = SwitchYqlTypeToArrowType(colType, [&]<typename TType>(TTypeWrapper<TType> typeHolder) { Y_UNUSED(typeHolder); @@ -1091,7 +1094,7 @@ bool TArrowToYdbConverter::Process(const arrow::RecordBatch& batch, TString& err }); if (!success) { - errorMessage = TStringBuilder() << "No arrow conversion for type Yql::" << NScheme::TypeName(colType) + errorMessage = TStringBuilder() << "No arrow conversion for type Yql::" << NScheme::TypeName(colType.GetTypeId()) << " at column '" << colName << "'"; return false; } @@ -1123,6 +1126,9 @@ bool TArrowToYdbConverter::Process(const arrow::RecordBatch& batch, TString& err ui32 col = 0; for (auto& [colName, colType] : YdbSchema) { + // TODO: support pg types + Y_VERIFY(colType.GetTypeId() != NScheme::NTypeIds::Pg, "pg types are not supported"); + auto& column = allColumns[col]; auto& curCell = cells[0][col]; if (column->IsNull(row)) { @@ -1137,7 +1143,7 @@ bool TArrowToYdbConverter::Process(const arrow::RecordBatch& batch, TString& err }); if (!success) { - errorMessage = TStringBuilder() << "No arrow conversion for type Yql::" << NScheme::TypeName(colType) + errorMessage = TStringBuilder() << "No arrow conversion for type Yql::" << NScheme::TypeName(colType.GetTypeId()) << " at column '" << colName << "'"; return false; } diff --git a/ydb/core/formats/arrow_helpers.h b/ydb/core/formats/arrow_helpers.h index bfe8fb88a3d..9f96d2b81b7 100644 --- a/ydb/core/formats/arrow_helpers.h +++ b/ydb/core/formats/arrow_helpers.h @@ -23,7 +23,7 @@ public: } }; -std::shared_ptr<arrow::DataType> GetArrowType(NScheme::TTypeId typeId); +std::shared_ptr<arrow::DataType> GetArrowType(NScheme::TTypeInfo typeInfo); template <typename T> inline bool ArrayEqualValue(const std::shared_ptr<arrow::Array>& x, const std::shared_ptr<arrow::Array>& y) { @@ -51,8 +51,8 @@ inline bool ArrayEqualView(const std::shared_ptr<arrow::Array>& x, const std::sh struct TSortDescription; -std::vector<std::shared_ptr<arrow::Field>> MakeArrowFields(const TVector<std::pair<TString, NScheme::TTypeId>>& columns); -std::shared_ptr<arrow::Schema> MakeArrowSchema(const TVector<std::pair<TString, NScheme::TTypeId>>& columns); +std::vector<std::shared_ptr<arrow::Field>> MakeArrowFields(const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns); +std::shared_ptr<arrow::Schema> MakeArrowSchema(const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns); TString SerializeSchema(const arrow::Schema& schema); std::shared_ptr<arrow::Schema> DeserializeSchema(const TString& str); @@ -154,7 +154,7 @@ class IRowWriter; // Converts an arrow batch into YDB rows feeding them IRowWriter one by one class TArrowToYdbConverter { private: - TVector<std::pair<TString, NScheme::TTypeId>> YdbSchema; // Destination schema (allow shrink and reorder) + TVector<std::pair<TString, NScheme::TTypeInfo>> YdbSchema; // Destination schema (allow shrink and reorder) IRowWriter& RowWriter; template <typename TArray> @@ -191,8 +191,8 @@ private: } public: - static bool NeedDataConversion(const NScheme::TTypeId& colType) { - switch (colType) { + static bool NeedDataConversion(const NScheme::TTypeInfo& colType) { + switch (colType.GetTypeId()) { case NScheme::NTypeIds::DyNumber: case NScheme::NTypeIds::JsonDocument: case NScheme::NTypeIds::Decimal: @@ -203,7 +203,7 @@ public: return false; } - TArrowToYdbConverter(const TVector<std::pair<TString, NScheme::TTypeId>>& ydbSchema, IRowWriter& rowWriter) + TArrowToYdbConverter(const TVector<std::pair<TString, NScheme::TTypeInfo>>& ydbSchema, IRowWriter& rowWriter) : YdbSchema(ydbSchema) , RowWriter(rowWriter) {} @@ -221,7 +221,7 @@ public: }; std::shared_ptr<arrow::RecordBatch> ConvertColumns(const std::shared_ptr<arrow::RecordBatch>& batch, - const THashMap<TString, NScheme::TTypeId>& columnsToConvert); + const THashMap<TString, NScheme::TTypeInfo>& columnsToConvert); inline bool HasNulls(const std::shared_ptr<arrow::Array>& column) { return column->null_bitmap_data(); diff --git a/ydb/core/formats/clickhouse_block.cpp b/ydb/core/formats/clickhouse_block.cpp index 0fd77f5fdad..2b5a629b517 100644 --- a/ydb/core/formats/clickhouse_block.cpp +++ b/ydb/core/formats/clickhouse_block.cpp @@ -440,13 +440,13 @@ public: return Types.at(name); } - TDataTypePtr GetByYdbType(NScheme::TTypeId t) const { + TDataTypePtr GetByYdbType(NScheme::TTypeInfo type) const { #define CONVERT(ydbType, chType) \ case NScheme::NTypeIds::ydbType: \ return Get(#chType); - switch (t) { + switch (type.GetTypeId()) { CONVERT(Bool, UInt8); CONVERT(Int8, Int8); @@ -479,8 +479,12 @@ public: CONVERT(ActorId, String); CONVERT(StepOrderId, String); + case NScheme::NTypeIds::Pg: + // TODO: support pg types + throw yexception() << "Unsupported pg type"; + default: - throw yexception() << "Unsupported type: " << t; + throw yexception() << "Unsupported type: " << type.GetTypeId(); } #undef CONVERT } @@ -533,13 +537,16 @@ void AddDecimal(const TMutableColumnPtr& column, const TCell& cell) { } } -size_t AddValue(const TMutableColumnPtr& column, const TCell& cell, ui32 ydbType) { - Y_UNUSED(ydbType); +size_t AddValue(const TMutableColumnPtr& column, const TCell& cell, NScheme::TTypeInfo type) { size_t prevBytes = column->byteSize(); if (cell.IsNull()) { AddNull(column); } else { - if (ydbType == NScheme::NTypeIds::Decimal) { + auto typeId = type.GetTypeId(); + if (typeId == NScheme::NTypeIds::Pg) { + // TODO: support pg types + Y_VERIFY(false, "pg types are not supported"); + } else if (typeId == NScheme::NTypeIds::Decimal) { AddDecimal(column, cell); } else { column->insertData(cell.Data(), cell.Size()); @@ -548,7 +555,7 @@ size_t AddValue(const TMutableColumnPtr& column, const TCell& cell, ui32 ydbType return column->byteSize() - prevBytes; } -TTypesAndNames MakeColumns(const TDataTypeRegistryPtr& dataTypeRegistry, const TVector<std::pair<TString, NScheme::TTypeId>>& columns) { +TTypesAndNames MakeColumns(const TDataTypeRegistryPtr& dataTypeRegistry, const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns) { TTypesAndNames res; for (auto& c : columns) { TDataTypePtr dataType = dataTypeRegistry->GetByYdbType(c.second); @@ -568,7 +575,7 @@ class TBlockBuilder : public NKikimr::IBlockBuilder { public: explicit TBlockBuilder(TDataTypeRegistryPtr dataTypeRegistry); ~TBlockBuilder(); - bool Start(const TVector<std::pair<TString, NScheme::TTypeId>>& columns, ui64 maxRowsInBlock, ui64 maxBytesInBlock, TString& err) override; + bool Start(const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns, ui64 maxRowsInBlock, ui64 maxBytesInBlock, TString& err) override; void AddRow(const NKikimr::TDbTupleRef& key, const NKikimr::TDbTupleRef& value) override; TString Finish() override; size_t Bytes() const override; @@ -587,7 +594,7 @@ private: class TBlockBuilder::TImpl { constexpr static ui32 DBMS_MIN_REVISION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD = 54408; public: - TImpl(const TDataTypeRegistryPtr& dataTypeRegistry, const TVector<std::pair<TString, NScheme::TTypeId>>& columns, ui64 maxRowsInBlock, ui64 maxBytesInBlock) + TImpl(const TDataTypeRegistryPtr& dataTypeRegistry, const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns, ui64 maxRowsInBlock, ui64 maxBytesInBlock) : MaxRowsInBlock(maxRowsInBlock) , MaxBytesInBlock(maxBytesInBlock) , BlockTemplate(MakeColumns(dataTypeRegistry, columns)) @@ -656,7 +663,7 @@ TBlockBuilder::TBlockBuilder(TDataTypeRegistryPtr dataTypeRegistry) TBlockBuilder::~TBlockBuilder() { } -bool TBlockBuilder::Start(const TVector<std::pair<TString, NScheme::TTypeId>>& columns, ui64 maxRowsInBlock, ui64 maxBytesInBlock, TString& err) { +bool TBlockBuilder::Start(const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns, ui64 maxRowsInBlock, ui64 maxBytesInBlock, TString& err) { try { Impl.Reset(new TImpl(DataTypeRegistry, columns, maxRowsInBlock, maxBytesInBlock)); } catch (std::exception& e) { diff --git a/ydb/core/formats/factory.h b/ydb/core/formats/factory.h index 59f9079fac7..40d44a12d4a 100644 --- a/ydb/core/formats/factory.h +++ b/ydb/core/formats/factory.h @@ -11,7 +11,7 @@ class IBlockBuilder { public: virtual ~IBlockBuilder() = default; - virtual bool Start(const TVector<std::pair<TString, NScheme::TTypeId>>& columns, ui64 maxRowsInBlock, ui64 maxBytesInBlock, TString& err) = 0; + virtual bool Start(const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns, ui64 maxRowsInBlock, ui64 maxBytesInBlock, TString& err) = 0; virtual void AddRow(const TDbTupleRef& key, const TDbTupleRef& value) = 0; virtual TString Finish() = 0; virtual size_t Bytes() const = 0; diff --git a/ydb/core/formats/switch_type.h b/ydb/core/formats/switch_type.h index 5cc7924389d..c04d59fd248 100644 --- a/ydb/core/formats/switch_type.h +++ b/ydb/core/formats/switch_type.h @@ -108,8 +108,8 @@ bool SwitchArrayType(const arrow::Datum& column, TFunc&& f) { * @return Result of execution of callback or false if the type typeId is not supported. */ template <typename TFunc> -bool SwitchYqlTypeToArrowType(NScheme::TTypeId typeId, TFunc&& callback) { - switch (typeId) { +bool SwitchYqlTypeToArrowType(NScheme::TTypeInfo typeInfo, TFunc&& callback) { + switch (typeInfo.GetTypeId()) { case NScheme::NTypeIds::Bool: return callback(TTypeWrapper<arrow::BooleanType>()); case NScheme::NTypeIds::Int8: @@ -155,6 +155,9 @@ bool SwitchYqlTypeToArrowType(NScheme::TTypeId typeId, TFunc&& callback) { case NScheme::NTypeIds::ActorId: case NScheme::NTypeIds::StepOrderId: break; // Deprecated types + + case NScheme::NTypeIds::Pg: + break; // TODO: support pg types } return false; } diff --git a/ydb/core/formats/ut_arrow.cpp b/ydb/core/formats/ut_arrow.cpp index 68f338b0205..d49c04fd389 100644 --- a/ydb/core/formats/ut_arrow.cpp +++ b/ydb/core/formats/ut_arrow.cpp @@ -14,29 +14,30 @@ namespace { namespace NTypeIds = NScheme::NTypeIds; using TTypeId = NScheme::TTypeId; +using TTypeInfo = NScheme::TTypeInfo; struct TDataRow { - static const constexpr TTypeId Types[20] = { - NTypeIds::Bool, - NTypeIds::Int8, - NTypeIds::Int16, - NTypeIds::Int32, - NTypeIds::Int64, - NTypeIds::Uint8, - NTypeIds::Uint16, - NTypeIds::Uint32, - NTypeIds::Uint64, - NTypeIds::Float, - NTypeIds::Double, - NTypeIds::String, - NTypeIds::Utf8, - NTypeIds::Json, - NTypeIds::Yson, - NTypeIds::Date, - NTypeIds::Datetime, - NTypeIds::Timestamp, - NTypeIds::Interval, - NTypeIds::JsonDocument, + static const constexpr TTypeInfo Types[20] = { + TTypeInfo(NTypeIds::Bool), + TTypeInfo(NTypeIds::Int8), + TTypeInfo(NTypeIds::Int16), + TTypeInfo(NTypeIds::Int32), + TTypeInfo(NTypeIds::Int64), + TTypeInfo(NTypeIds::Uint8), + TTypeInfo(NTypeIds::Uint16), + TTypeInfo(NTypeIds::Uint32), + TTypeInfo(NTypeIds::Uint64), + TTypeInfo(NTypeIds::Float), + TTypeInfo(NTypeIds::Double), + TTypeInfo(NTypeIds::String), + TTypeInfo(NTypeIds::Utf8), + TTypeInfo(NTypeIds::Json), + TTypeInfo(NTypeIds::Yson), + TTypeInfo(NTypeIds::Date), + TTypeInfo(NTypeIds::Datetime), + TTypeInfo(NTypeIds::Timestamp), + TTypeInfo(NTypeIds::Interval), + TTypeInfo(NTypeIds::JsonDocument), // TODO: DyNumber, Decimal }; @@ -114,29 +115,29 @@ struct TDataRow { return std::make_shared<arrow::Schema>(fields); } - static TVector<std::pair<TString, TTypeId>> MakeYdbSchema() { - TVector<std::pair<TString, TTypeId>> columns = { - {"bool", NTypeIds::Bool }, - {"i8", NTypeIds::Int8 }, - {"i16", NTypeIds::Int16 }, - {"i32", NTypeIds::Int32 }, - {"i64", NTypeIds::Int64 }, - {"ui8", NTypeIds::Uint8 }, - {"ui16", NTypeIds::Uint16 }, - {"ui32", NTypeIds::Uint32 }, - {"ui64", NTypeIds::Uint64 }, - {"f32", NTypeIds::Float }, - {"f64", NTypeIds::Double }, - {"string", NTypeIds::String }, - {"utf8", NTypeIds::Utf8 }, - {"json", NTypeIds::Json }, - {"yson", NTypeIds::Yson }, - {"date", NTypeIds::Date }, - {"datetime", NTypeIds::Datetime }, - {"ts", NTypeIds::Timestamp }, - {"ival", NTypeIds::Interval }, - {"json_doc", NTypeIds::JsonDocument }, - //{"dec", NTypeIds::Decimal } + static TVector<std::pair<TString, TTypeInfo>> MakeYdbSchema() { + TVector<std::pair<TString, TTypeInfo>> columns = { + {"bool", TTypeInfo(NTypeIds::Bool) }, + {"i8", TTypeInfo(NTypeIds::Int8) }, + {"i16", TTypeInfo(NTypeIds::Int16) }, + {"i32", TTypeInfo(NTypeIds::Int32) }, + {"i64", TTypeInfo(NTypeIds::Int64) }, + {"ui8", TTypeInfo(NTypeIds::Uint8) }, + {"ui16", TTypeInfo(NTypeIds::Uint16) }, + {"ui32", TTypeInfo(NTypeIds::Uint32) }, + {"ui64", TTypeInfo(NTypeIds::Uint64) }, + {"f32", TTypeInfo(NTypeIds::Float) }, + {"f64", TTypeInfo(NTypeIds::Double) }, + {"string", TTypeInfo(NTypeIds::String) }, + {"utf8", TTypeInfo(NTypeIds::Utf8) }, + {"json", TTypeInfo(NTypeIds::Json) }, + {"yson", TTypeInfo(NTypeIds::Yson) }, + {"date", TTypeInfo(NTypeIds::Date) }, + {"datetime", TTypeInfo(NTypeIds::Datetime) }, + {"ts", TTypeInfo(NTypeIds::Timestamp) }, + {"ival", TTypeInfo(NTypeIds::Interval) }, + {"json_doc", TTypeInfo(NTypeIds::JsonDocument) }, + //{"dec", TTypeInfo(NTypeIds::Decimal) } }; return columns; } diff --git a/ydb/core/grpc_services/rpc_import_data.cpp b/ydb/core/grpc_services/rpc_import_data.cpp index d9b32d742a5..33bd25942ab 100644 --- a/ydb/core/grpc_services/rpc_import_data.cpp +++ b/ydb/core/grpc_services/rpc_import_data.cpp @@ -42,8 +42,8 @@ class TImportDataRPC: public TRpcRequestActor<TImportDataRPC, TEvImportDataReque static constexpr TDuration MAX_TIMEOUT = TDuration::Minutes(5); - static TVector<NUdf::TDataTypeId> MakeKeyColumnTypes(const TNavigate::TEntry& entry) { - TVector<NUdf::TDataTypeId> result; + static TVector<NScheme::TTypeInfo> MakeKeyColumnTypes(const TNavigate::TEntry& entry) { + TVector<NScheme::TTypeInfo> result; for (const auto& [_, column] : entry.Columns) { if (column.KeyOrder < 0) { @@ -67,7 +67,7 @@ class TImportDataRPC: public TRpcRequestActor<TImportDataRPC, TEvImportDataReque } static THolder<TKeyDesc> MakeKeyDesc(const TNavigate::TEntry& entry) { - const TVector<NUdf::TDataTypeId> keyColumnTypes = MakeKeyColumnTypes(entry); + const TVector<NScheme::TTypeInfo> keyColumnTypes = MakeKeyColumnTypes(entry); return MakeHolder<TKeyDesc>( entry.TableId, GetFullRange(keyColumnTypes.size()).ToTableRange(), @@ -290,7 +290,7 @@ class TImportDataRPC: public TRpcRequestActor<TImportDataRPC, TEvImportDataReque TMaybe<TOwnedCellVec> prevKey; TMemoryPool pool(256); - std::vector<std::pair<i32, ui32>> columnOrderTypes; // {keyOrder, PType} + std::vector<std::pair<i32, NScheme::TTypeInfo>> columnOrderTypes; // {keyOrder, PType} columnOrderTypes.reserve(format.columns().size()); for (const auto& column : format.columns()) { const auto* info = Columns.FindPtr(column); diff --git a/ydb/core/grpc_services/rpc_kh_describe.cpp b/ydb/core/grpc_services/rpc_kh_describe.cpp index 9882b0abb1a..2cd5434a926 100644 --- a/ydb/core/grpc_services/rpc_kh_describe.cpp +++ b/ydb/core/grpc_services/rpc_kh_describe.cpp @@ -42,7 +42,7 @@ private: bool WaitingResolveReply; bool Finished; - TVector<NScheme::TTypeId> KeyColumnTypes; + TVector<NScheme::TTypeInfo> KeyColumnTypes; THolder<NKikimr::TKeyDesc> KeyRange; TAutoPtr<NSchemeCache::TSchemeCacheNavigate> ResolveNamesResult; @@ -175,7 +175,13 @@ private: for (const auto& col : entry.Columns) { auto* colMeta = Result.add_columns(); colMeta->set_name(col.second.Name); - colMeta->mutable_type()->mutable_optional_type()->mutable_item()->set_type_id((Ydb::Type::PrimitiveTypeId)col.second.PType); + auto& typeInfo = col.second.PType; + auto* item = colMeta->mutable_type()->mutable_optional_type()->mutable_item(); + if (typeInfo.GetTypeId() == NScheme::NTypeIds::Pg) { + item->mutable_pg_type()->set_oid(NPg::PgTypeIdFromTypeDesc(typeInfo.GetTypeDesc())); + } else { + item->set_type_id((Ydb::Type::PrimitiveTypeId)typeInfo.GetTypeId()); + } if (col.second.KeyOrder == -1) continue; diff --git a/ydb/core/grpc_services/rpc_load_rows.cpp b/ydb/core/grpc_services/rpc_load_rows.cpp index be234b7eff6..7cdb15f5ab7 100644 --- a/ydb/core/grpc_services/rpc_load_rows.cpp +++ b/ydb/core/grpc_services/rpc_load_rows.cpp @@ -25,9 +25,9 @@ using namespace Ydb; namespace { -bool CheckValueData(NScheme::TTypeId type, const TCell& cell, TString& err) { +bool CheckValueData(NScheme::TTypeInfo type, const TCell& cell, TString& err) { bool ok = true; - switch (type) { + switch (type.GetTypeId()) { case NScheme::NTypeIds::Bool: case NScheme::NTypeIds::Int8: case NScheme::NTypeIds::Uint8: @@ -82,8 +82,12 @@ bool CheckValueData(NScheme::TTypeId type, const TCell& cell, TString& err) { // DyNumber value was verified at parsing time break; + case NScheme::NTypeIds::Pg: + // no pg validation here + break; + default: - err = Sprintf("Unexpected type %d", type); + err = Sprintf("Unexpected type %d", type.GetTypeId()); return false; } @@ -191,7 +195,7 @@ public: {} private: - static bool CellFromProtoVal(NScheme::TTypeId type, const Ydb::Value* vp, + static bool CellFromProtoVal(NScheme::TTypeInfo type, const Ydb::Value* vp, TCell& c, TString& err, TMemoryPool& valueDataPool) { if (vp->Hasnull_flag_value()) { @@ -212,7 +216,7 @@ private: break; \ } - switch (type) { + switch (type.GetTypeId()) { EXTRACT_VAL(Bool, bool, ui8); EXTRACT_VAL(Int8, int32, i8); EXTRACT_VAL(Uint8, uint32, ui8); @@ -267,8 +271,13 @@ private: c = TCell((const char*)&decimalVal, sizeof(decimalVal)); break; } + case NScheme::NTypeIds::Pg : { + TString v = val.Getbytes_value(); + c = TCell(v.data(), v.size()); + break; + } default: - err = Sprintf("Unexpected type %d", type); + err = Sprintf("Unexpected type %d", type.GetTypeId()); return false; }; diff --git a/ydb/core/grpc_services/rpc_log_store.cpp b/ydb/core/grpc_services/rpc_log_store.cpp index e82b54488fc..732303407bc 100644 --- a/ydb/core/grpc_services/rpc_log_store.cpp +++ b/ydb/core/grpc_services/rpc_log_store.cpp @@ -95,15 +95,15 @@ bool ConvertSchemaFromPublicToInternal(const Ydb::LogStore::Schema& from, NKikim for (const auto& column : from.columns()) { auto* col = to.AddColumns(); col->SetName(column.name()); - ui32 typeId; - if (!ExtractColumnTypeId(typeId, column.type(), status, error)) { + NScheme::TTypeInfo typeInfo; + if (!ExtractColumnTypeInfo(typeInfo, column.type(), status, error)) { return false; } - auto typeName = NScheme::TypeName(typeId); + auto typeName = NScheme::TypeName(typeInfo); col->SetType(typeName); key.erase(column.name()); - if (column.name() == firstKeyColumn && typeId != NYql::NProto::Timestamp) { + if (column.name() == firstKeyColumn && typeInfo.GetTypeId() != NYql::NProto::Timestamp) { error = "not supported first PK column type for LogStore. Only Timestamp columns are allowed for now."; return false; } diff --git a/ydb/core/grpc_services/rpc_long_tx.cpp b/ydb/core/grpc_services/rpc_long_tx.cpp index 276a500ea72..3ca643789a0 100644 --- a/ydb/core/grpc_services/rpc_long_tx.cpp +++ b/ydb/core/grpc_services/rpc_long_tx.cpp @@ -10,6 +10,7 @@ #include <ydb/core/tablet/tablet_pipe_client_cache.h> #include <ydb/core/formats/arrow_helpers.h> #include <ydb/core/formats/sharding.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/tx/columnshard/columnshard.h> #include <ydb/core/tx/long_tx_service/public/events.h> @@ -33,10 +34,12 @@ using TEvLongTxReadRequest = NGRpcService::TGrpcRequestOperationCall<Ydb::LongTx Ydb::LongTx::ReadResponse>; std::shared_ptr<arrow::Schema> ExtractArrowSchema(const NKikimrSchemeOp::TColumnTableSchema& schema) { - TVector<std::pair<TString, NScheme::TTypeId>> columns; + TVector<std::pair<TString, NScheme::TTypeInfo>> columns; for (auto& col : schema.GetColumns()) { Y_VERIFY(col.HasTypeId()); - columns.emplace_back(col.GetName(), (NScheme::TTypeId)col.GetTypeId()); + auto typeInfo = NScheme::TypeInfoFromProtoColumnType(col.GetTypeId(), + col.HasTypeInfo() ? &col.GetTypeInfo() : nullptr); + columns.emplace_back(col.GetName(), typeInfo); } return NArrow::MakeArrowSchema(columns); diff --git a/ydb/core/grpc_services/rpc_read_columns.cpp b/ydb/core/grpc_services/rpc_read_columns.cpp index d9f17687a94..bbff1078e6b 100644 --- a/ydb/core/grpc_services/rpc_read_columns.cpp +++ b/ydb/core/grpc_services/rpc_read_columns.cpp @@ -43,13 +43,13 @@ private: TAutoPtr<TKeyDesc> KeyRange; TAutoPtr<NSchemeCache::TSchemeCacheNavigate> ResolveNamesResult; - TVector<NScheme::TTypeId> KeyColumnTypes; + TVector<NScheme::TTypeInfo> KeyColumnTypes; // Positions of key and value fields in the request proto struct struct TFieldDescription { ui32 ColId; ui32 PositionInStruct; - NScheme::TTypeId Type; + NScheme::TTypeInfo Type; }; TVector<TFieldDescription> KeyColumnPositions; TVector<TFieldDescription> ValueColumnPositions; @@ -64,7 +64,7 @@ private: TActorId SysViewScanActor; std::unique_ptr<IBlockBuilder> BlockBuilder; - TVector<NScheme::TTypeId> ValueColumnTypes; + TVector<NScheme::TTypeInfo> ValueColumnTypes; ui64 SysViewMaxRows; ui64 SysViewMaxBytes; ui64 SysViewRowsReceived; @@ -254,7 +254,7 @@ private: SysViewMaxBytes = maxBytes; // List of columns requested by user - TVector<std::pair<TString, NScheme::TTypeId>> valueColumnNamesAndTypes; + TVector<std::pair<TString, NScheme::TTypeInfo>> valueColumnNamesAndTypes; // This list of columns will be requested from sys view scan actor // It starts with all key columns followed by all the columns requested by user possibly including key columns again @@ -270,7 +270,7 @@ private: KeyColumnTypes[ci.second.KeyOrder] = ci.second.PType; columns.resize(Max<size_t>(columns.size(), ci.second.KeyOrder + 1)); - columns[ci.second.KeyOrder] = {ci.second.Id, (NScheme::TTypeId)ci.second.PType}; + columns[ci.second.KeyOrder] = {ci.second.Id, ci.second.PType}; } } @@ -283,10 +283,10 @@ private: } auto ci = entry.Columns.find(id->second); - columns.push_back({ci->second.Id, (NScheme::TTypeId)ci->second.PType}); + columns.push_back({ci->second.Id, ci->second.PType}); - valueColumnNamesAndTypes.push_back({ci->second.Name, (NScheme::TTypeId)ci->second.PType}); - ValueColumnTypes.push_back((NScheme::TTypeId)ci->second.PType); + valueColumnNamesAndTypes.push_back({ci->second.Name, ci->second.PType}); + ValueColumnTypes.push_back(ci->second.PType); } } @@ -505,7 +505,7 @@ private: return true; } - bool CheckCellSizes(const TConstArrayRef<TCell>& cells, const TConstArrayRef<NScheme::TTypeId>& types) { + bool CheckCellSizes(const TConstArrayRef<TCell>& cells, const TConstArrayRef<NScheme::TTypeInfo>& types) { if (cells.size() > types.size()) return false; diff --git a/ydb/core/grpc_services/ut/CMakeLists.darwin.txt b/ydb/core/grpc_services/ut/CMakeLists.darwin.txt index 573ff572497..10d02da6dfb 100644 --- a/ydb/core/grpc_services/ut/CMakeLists.darwin.txt +++ b/ydb/core/grpc_services/ut/CMakeLists.darwin.txt @@ -21,7 +21,7 @@ target_link_libraries(ydb-core-grpc_services-ut PUBLIC cpp-regex-pcre library-cpp-svnversion core-client-scheme_cache_lib - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-grpc_services-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/grpc_services/ut/CMakeLists.linux.txt b/ydb/core/grpc_services/ut/CMakeLists.linux.txt index 79560ac6a68..ead6d659edc 100644 --- a/ydb/core/grpc_services/ut/CMakeLists.linux.txt +++ b/ydb/core/grpc_services/ut/CMakeLists.linux.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-grpc_services-ut PUBLIC cpp-regex-pcre library-cpp-svnversion core-client-scheme_cache_lib - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-grpc_services-ut PRIVATE -ldl diff --git a/ydb/core/grpc_streaming/ut/CMakeLists.darwin.txt b/ydb/core/grpc_streaming/ut/CMakeLists.darwin.txt index 9f9cf5dae44..bbeafa0ac05 100644 --- a/ydb/core/grpc_streaming/ut/CMakeLists.darwin.txt +++ b/ydb/core/grpc_streaming/ut/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-grpc_streaming-ut PUBLIC ydb-core-grpc_streaming cpp-grpc-client grpc_streaming-ut-grpc - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-grpc_streaming-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/grpc_streaming/ut/CMakeLists.linux.txt b/ydb/core/grpc_streaming/ut/CMakeLists.linux.txt index 61213521677..18ffe471426 100644 --- a/ydb/core/grpc_streaming/ut/CMakeLists.linux.txt +++ b/ydb/core/grpc_streaming/ut/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-grpc_streaming-ut PUBLIC ydb-core-grpc_streaming cpp-grpc-client grpc_streaming-ut-grpc - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-grpc_streaming-ut PRIVATE -ldl diff --git a/ydb/core/health_check/ut/CMakeLists.darwin.txt b/ydb/core/health_check/ut/CMakeLists.darwin.txt index fbfdfed6dfc..2b7b62513ff 100644 --- a/ydb/core/health_check/ut/CMakeLists.darwin.txt +++ b/ydb/core/health_check/ut/CMakeLists.darwin.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-core-health_check-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main ydb-core-health_check - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-health_check-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/health_check/ut/CMakeLists.linux.txt b/ydb/core/health_check/ut/CMakeLists.linux.txt index 9d64a700e31..c941884da7f 100644 --- a/ydb/core/health_check/ut/CMakeLists.linux.txt +++ b/ydb/core/health_check/ut/CMakeLists.linux.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-health_check-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main ydb-core-health_check - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-health_check-ut PRIVATE -ldl diff --git a/ydb/core/io_formats/csv.h b/ydb/core/io_formats/csv.h index d3e75a487df..0924ac16850 100644 --- a/ydb/core/io_formats/csv.h +++ b/ydb/core/io_formats/csv.h @@ -13,7 +13,7 @@ namespace NKikimr::NFormats { struct TYdbDump { // Parse YdbDump-formatted line - static bool ParseLine(TStringBuf line, const std::vector<std::pair<i32, ui32>>& columnOrderTypes, TMemoryPool& pool, + static bool ParseLine(TStringBuf line, const std::vector<std::pair<i32, NScheme::TTypeInfo>>& columnOrderTypes, TMemoryPool& pool, TVector<TCell>& keys, TVector<TCell>& values, TString& strError, ui64& numBytes); }; @@ -23,7 +23,7 @@ public: /// If header is true read column names from first line after skipRows. Parse columns as strings in this case. /// @note It's possible to skip header with skipRows and use typed columns instead. - TArrowCSV(const TVector<std::pair<TString, NScheme::TTypeId>>& columns, + TArrowCSV(const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns, ui32 skipRows = 0, bool header = false, ui32 blockSize = DEFAULT_BLOCK_SIZE); std::shared_ptr<arrow::RecordBatch> ReadNext(const TString& csv, TString& errString); diff --git a/ydb/core/io_formats/csv_arrow.cpp b/ydb/core/io_formats/csv_arrow.cpp index 6a79825e56d..e4a42f7f802 100644 --- a/ydb/core/io_formats/csv_arrow.cpp +++ b/ydb/core/io_formats/csv_arrow.cpp @@ -3,7 +3,7 @@ namespace NKikimr::NFormats { -TArrowCSV::TArrowCSV(const TVector<std::pair<TString, NScheme::TTypeId>>& columns, ui32 skipRows, bool header, +TArrowCSV::TArrowCSV(const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns, ui32 skipRows, bool header, ui32 blockSize) : ReadOptions(arrow::csv::ReadOptions::Defaults()) , ParseOptions(arrow::csv::ParseOptions::Defaults()) diff --git a/ydb/core/io_formats/csv_ydb_dump.cpp b/ydb/core/io_formats/csv_ydb_dump.cpp index 683a55a0856..5aab7d3b1d6 100644 --- a/ydb/core/io_formats/csv_ydb_dump.cpp +++ b/ydb/core/io_formats/csv_ydb_dump.cpp @@ -164,12 +164,12 @@ namespace { } // anonymous -bool MakeCell(TCell& cell, TStringBuf value, NScheme::TTypeId type, TMemoryPool& pool) { +bool MakeCell(TCell& cell, TStringBuf value, NScheme::TTypeInfo type, TMemoryPool& pool) { if (value == "null") { return true; } - switch (type) { + switch (type.GetTypeId()) { case NScheme::NTypeIds::Bool: return TCellMaker<bool>::Make(cell, value, pool); case NScheme::NTypeIds::Int8: @@ -213,17 +213,20 @@ bool MakeCell(TCell& cell, TStringBuf value, NScheme::TTypeId type, TMemoryPool& return TCellMaker<TMaybe<TString>, TStringBuf>::Make(cell, value, pool, &DyNumberToStringBuf); case NScheme::NTypeIds::Decimal: return TCellMaker<NYql::NDecimal::TInt128, std::pair<ui64, ui64>>::Make(cell, value, pool, &Int128ToPair); + case NScheme::NTypeIds::Pg: + // TODO: support pg types + Y_VERIFY(false, "pg types are not supported"); default: return false; } } -bool CheckCellValue(const TCell& cell, NScheme::TTypeId type) { +bool CheckCellValue(const TCell& cell, NScheme::TTypeInfo type) { if (cell.IsNull()) { return true; } - switch (type) { + switch (type.GetTypeId()) { case NScheme::NTypeIds::Bool: case NScheme::NTypeIds::Int8: case NScheme::NTypeIds::Uint8: @@ -257,12 +260,15 @@ bool CheckCellValue(const TCell& cell, NScheme::TTypeId type) { return NYql::NDom::IsValidJson(cell.AsBuf()); case NScheme::NTypeIds::Decimal: return !NYql::NDecimal::IsError(cell.AsValue<NYql::NDecimal::TInt128>()); + case NScheme::NTypeIds::Pg: + // TODO: support pg types + Y_VERIFY(false, "pg types are not supported"); default: return false; } } -bool TYdbDump::ParseLine(TStringBuf line, const std::vector<std::pair<i32, ui32>>& columnOrderTypes, TMemoryPool& pool, +bool TYdbDump::ParseLine(TStringBuf line, const std::vector<std::pair<i32, NScheme::TTypeInfo>>& columnOrderTypes, TMemoryPool& pool, TVector<TCell>& keys, TVector<TCell>& values, TString& strError, ui64& numBytes) { for (const auto& [keyOrder, pType] : columnOrderTypes) { diff --git a/ydb/core/kesus/proxy/ut/CMakeLists.darwin.txt b/ydb/core/kesus/proxy/ut/CMakeLists.darwin.txt index c4258c7c46a..6d97c9882f6 100644 --- a/ydb/core/kesus/proxy/ut/CMakeLists.darwin.txt +++ b/ydb/core/kesus/proxy/ut/CMakeLists.darwin.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-core-kesus-proxy-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main core-kesus-proxy - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-kesus-proxy-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/kesus/proxy/ut/CMakeLists.linux.txt b/ydb/core/kesus/proxy/ut/CMakeLists.linux.txt index 4d415bbecb4..85fdeb1b574 100644 --- a/ydb/core/kesus/proxy/ut/CMakeLists.linux.txt +++ b/ydb/core/kesus/proxy/ut/CMakeLists.linux.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-kesus-proxy-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main core-kesus-proxy - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-kesus-proxy-ut PRIVATE -ldl diff --git a/ydb/core/kesus/tablet/quoter_performance_test/CMakeLists.darwin.txt b/ydb/core/kesus/tablet/quoter_performance_test/CMakeLists.darwin.txt index 9df85c22844..19fec0b0b0a 100644 --- a/ydb/core/kesus/tablet/quoter_performance_test/CMakeLists.darwin.txt +++ b/ydb/core/kesus/tablet/quoter_performance_test/CMakeLists.darwin.txt @@ -21,7 +21,7 @@ target_link_libraries(quoter_performance_test PUBLIC library-cpp-getopt cpp-testing-unittest core-kesus-tablet - ydb-core-testlib + core-testlib-default ) target_link_options(quoter_performance_test PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/kesus/tablet/quoter_performance_test/CMakeLists.linux.txt b/ydb/core/kesus/tablet/quoter_performance_test/CMakeLists.linux.txt index ba3470a24ee..1917fedb4a1 100644 --- a/ydb/core/kesus/tablet/quoter_performance_test/CMakeLists.linux.txt +++ b/ydb/core/kesus/tablet/quoter_performance_test/CMakeLists.linux.txt @@ -23,7 +23,7 @@ target_link_libraries(quoter_performance_test PUBLIC library-cpp-getopt cpp-testing-unittest core-kesus-tablet - ydb-core-testlib + core-testlib-default ) target_link_options(quoter_performance_test PRIVATE -ldl diff --git a/ydb/core/kesus/tablet/ut/CMakeLists.darwin.txt b/ydb/core/kesus/tablet/ut/CMakeLists.darwin.txt index 3c3ea0c5d72..ea79b18cc98 100644 --- a/ydb/core/kesus/tablet/ut/CMakeLists.darwin.txt +++ b/ydb/core/kesus/tablet/ut/CMakeLists.darwin.txt @@ -21,7 +21,7 @@ target_link_libraries(ydb-core-kesus-tablet-ut PUBLIC cpp-testing-unittest_main core-kesus-tablet cpp-testing-gmock_in_unittest - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-kesus-tablet-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/kesus/tablet/ut/CMakeLists.linux.txt b/ydb/core/kesus/tablet/ut/CMakeLists.linux.txt index 6cc921b804d..c9d5fdfc69a 100644 --- a/ydb/core/kesus/tablet/ut/CMakeLists.linux.txt +++ b/ydb/core/kesus/tablet/ut/CMakeLists.linux.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-kesus-tablet-ut PUBLIC cpp-testing-unittest_main core-kesus-tablet cpp-testing-gmock_in_unittest - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-kesus-tablet-ut PRIVATE -ldl diff --git a/ydb/core/keyvalue/ut/CMakeLists.darwin.txt b/ydb/core/keyvalue/ut/CMakeLists.darwin.txt index 309ef9b9fc8..cfcb539eb06 100644 --- a/ydb/core/keyvalue/ut/CMakeLists.darwin.txt +++ b/ydb/core/keyvalue/ut/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-keyvalue-ut PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-keyvalue-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/keyvalue/ut/CMakeLists.linux.txt b/ydb/core/keyvalue/ut/CMakeLists.linux.txt index 361058b8dc9..32835181c8c 100644 --- a/ydb/core/keyvalue/ut/CMakeLists.linux.txt +++ b/ydb/core/keyvalue/ut/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-keyvalue-ut PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-keyvalue-ut PRIVATE -ldl diff --git a/ydb/core/kqp/common/CMakeLists.txt b/ydb/core/kqp/common/CMakeLists.txt index f07836ba559..c4926bd4b0f 100644 --- a/ydb/core/kqp/common/CMakeLists.txt +++ b/ydb/core/kqp/common/CMakeLists.txt @@ -35,6 +35,7 @@ target_sources(core-kqp-common PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/kqp/common/kqp_timeouts.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/common/kqp_lwtrace_probes.cpp ${CMAKE_SOURCE_DIR}/ydb/core/kqp/common/kqp_topic.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/common/kqp_types.cpp ) generate_enum_serilization(core-kqp-common ${CMAKE_SOURCE_DIR}/ydb/core/kqp/common/kqp_tx_info.h diff --git a/ydb/core/kqp/common/kqp_resolve.cpp b/ydb/core/kqp/common/kqp_resolve.cpp index b4abafc86c6..64dac311215 100644 --- a/ydb/core/kqp/common/kqp_resolve.cpp +++ b/ydb/core/kqp/common/kqp_resolve.cpp @@ -16,7 +16,7 @@ using namespace NMiniKQL; using namespace NYql; using namespace NYql::NNodes; -TVector<TCell> MakeKeyCells(const NKikimr::NUdf::TUnboxedValue& value, const TVector<NUdf::TDataTypeId>& keyColumnTypes, +TVector<TCell> MakeKeyCells(const NKikimr::NUdf::TUnboxedValue& value, const TVector<NScheme::TTypeInfo>& keyColumnTypes, const TVector<ui32>& keyColumnIndices, const NMiniKQL::TTypeEnvironment& typeEnv, bool copyValues) { TVector<TCell> key(keyColumnTypes.size()); @@ -47,7 +47,7 @@ TTableId MakeTableId(const NKqpProto::TKqpPhyTable& table) { } TVector<TPartitionWithRange> GetKeyRangePartitions(const TTableRange& range, - const TVector<TKeyDesc::TPartitionInfo>& partitions, const TVector<NUdf::TDataTypeId>& keyColumnTypes) + const TVector<TKeyDesc::TPartitionInfo>& partitions, const TVector<NScheme::TTypeInfo>& keyColumnTypes) { auto it = std::lower_bound(partitions.begin(), partitions.end(), true, [&range, &keyColumnTypes](const auto& partition, bool) { diff --git a/ydb/core/kqp/common/kqp_resolve.h b/ydb/core/kqp/common/kqp_resolve.h index 517d1bc451d..f2f34e2f8a4 100644 --- a/ydb/core/kqp/common/kqp_resolve.h +++ b/ydb/core/kqp/common/kqp_resolve.h @@ -23,14 +23,14 @@ class TKqpTableKeys { public: struct TColumn { ui32 Id; - ui32 Type; + NScheme::TTypeInfo Type; }; struct TTable { TString Path; TMap<TString, TColumn> Columns; TVector<TString> KeyColumns; - TVector<NUdf::TDataTypeId> KeyColumnTypes; + TVector<NScheme::TTypeInfo> KeyColumnTypes; ETableKind TableKind = ETableKind::Unknown; }; @@ -72,12 +72,12 @@ private: THashMap<TTableId, TTable> TablesById; }; -TVector<TCell> MakeKeyCells(const NKikimr::NUdf::TUnboxedValue& value, const TVector<NUdf::TDataTypeId>& keyColumnTypes, +TVector<TCell> MakeKeyCells(const NKikimr::NUdf::TUnboxedValue& value, const TVector<NScheme::TTypeInfo>& keyColumnTypes, const TVector<ui32>& keyColumnIndices, const NMiniKQL::TTypeEnvironment& typeEnv, bool copyValues); template<typename TList, typename TRangeFunc> size_t FindKeyPartitionIndex(const TVector<TCell>& key, const TList& partitions, - const TVector<NUdf::TDataTypeId>& keyColumnTypes, const TRangeFunc& rangeFunc) + const TVector<NScheme::TTypeInfo>& keyColumnTypes, const TRangeFunc& rangeFunc) { auto it = std::lower_bound(partitions.begin(), partitions.end(), key, [&keyColumnTypes, &rangeFunc](const auto& partition, const auto& key) { @@ -95,7 +95,7 @@ size_t FindKeyPartitionIndex(const TVector<TCell>& key, const TList& partitions, template<typename TList, typename TRangeFunc> size_t FindKeyPartitionIndex(const NMiniKQL::TTypeEnvironment& typeEnv, const NKikimr::NUdf::TUnboxedValue& value, - const TList& partitions, const TVector<NUdf::TDataTypeId>& keyColumnTypes, const TVector<ui32>& keyColumnIndices, + const TList& partitions, const TVector<NScheme::TTypeInfo>& keyColumnTypes, const TVector<ui32>& keyColumnIndices, const TRangeFunc& rangeFunc) { auto key = MakeKeyCells(value, keyColumnTypes, keyColumnIndices, typeEnv, /* copyValues */ true); @@ -115,10 +115,10 @@ struct TPartitionWithRange { }; TVector<TPartitionWithRange> GetKeyRangePartitions(const TTableRange& range, - const TVector<TKeyDesc::TPartitionInfo>& partitions, const TVector<NUdf::TDataTypeId>& keyColumnTypes); + const TVector<TKeyDesc::TPartitionInfo>& partitions, const TVector<NScheme::TTypeInfo>& keyColumnTypes); template<typename TList, typename TRangeFunc> -void SortPartitions(TList& partitions, const TVector<NUdf::TDataTypeId>& keyColumnTypes, const TRangeFunc& rangeFunc) { +void SortPartitions(TList& partitions, const TVector<NScheme::TTypeInfo>& keyColumnTypes, const TRangeFunc& rangeFunc) { std::sort(partitions.begin(), partitions.end(), [&keyColumnTypes, &rangeFunc](const auto& left, const auto& right) { const auto& leftRange = rangeFunc(left); diff --git a/ydb/core/kqp/common/kqp_types.cpp b/ydb/core/kqp/common/kqp_types.cpp new file mode 100644 index 00000000000..22aed7aa5ef --- /dev/null +++ b/ydb/core/kqp/common/kqp_types.cpp @@ -0,0 +1,49 @@ +#include "kqp_types.h" + +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/utils/pg_types.h> + +namespace NKikimr::NScheme { + +void ProtoMiniKQLTypeFromTypeInfo(NKikimrMiniKQL::TType* type, const TTypeInfo typeInfo) { + if (typeInfo.GetTypeId() == NTypeIds::Pg) { + type->SetKind(NKikimrMiniKQL::Pg); + type->MutablePg()->Setoid(NPg::PgTypeIdFromTypeDesc(typeInfo.GetTypeDesc())); + } else { + type->SetKind(NKikimrMiniKQL::Data); + type->MutableData()->SetScheme(typeInfo.GetTypeId()); + } +} + +TTypeInfo TypeInfoFromProtoMiniKQLType(const NKikimrMiniKQL::TType& type) { + switch (type.GetKind()) { + case NKikimrMiniKQL::Data: + return TTypeInfo((NScheme::TTypeId)type.GetData().GetScheme()); + case NKikimrMiniKQL::Pg: + return TTypeInfo(NScheme::NTypeIds::Pg, NPg::TypeDescFromPgTypeId(type.GetPg().Getoid())); + default: + Y_ENSURE(false, "not a data or pg type"); + } +} + +const NMiniKQL::TType* MiniKQLTypeFromTypeInfo(const TTypeInfo typeInfo, const NMiniKQL::TTypeEnvironment& env) { + if (typeInfo.GetTypeId() == NTypeIds::Pg) { + return NMiniKQL::TPgType::Create(NPg::PgTypeIdFromTypeDesc(typeInfo.GetTypeDesc()), env); + } else { + return NMiniKQL::TDataType::Create((NUdf::TDataTypeId)typeInfo.GetTypeId(), env); + } +} + +TTypeInfo TypeInfoFromMiniKQLType(const NMiniKQL::TType* type) { + switch (type->GetKind()) { + case NMiniKQL::TType::EKind::Data: + return TTypeInfo((NScheme::TTypeId)AS_TYPE(NMiniKQL::TDataType, type)->GetSchemeType()); + case NMiniKQL::TType::EKind::Pg: + return TTypeInfo(NScheme::NTypeIds::Pg, NPg::TypeDescFromPgTypeId(AS_TYPE(NMiniKQL::TPgType, type)->GetTypeId())); + default: + Y_ENSURE(false, "not a data or pg type"); + } +} + +} // namespace NKikimr::NScheme + diff --git a/ydb/core/kqp/common/kqp_types.h b/ydb/core/kqp/common/kqp_types.h new file mode 100644 index 00000000000..1f2cc41cdd7 --- /dev/null +++ b/ydb/core/kqp/common/kqp_types.h @@ -0,0 +1,16 @@ +#pragma once + +#include <ydb/core/scheme_types/scheme_type_info.h> +#include <ydb/library/mkql_proto/protos/minikql.pb.h> +#include <ydb/library/yql/minikql/mkql_node.h> + +namespace NKikimr::NScheme { + +void ProtoMiniKQLTypeFromTypeInfo(NKikimrMiniKQL::TType* type, const TTypeInfo typeInfo); +TTypeInfo TypeInfoFromProtoMiniKQLType(const NKikimrMiniKQL::TType& type); + +const NMiniKQL::TType* MiniKQLTypeFromTypeInfo(const TTypeInfo typeInfo, + const NMiniKQL::TTypeEnvironment& env); +TTypeInfo TypeInfoFromMiniKQLType(const NMiniKQL::TType* type); + +} // namespace NKikimr::NScheme diff --git a/ydb/core/kqp/compile/kqp_compile.cpp b/ydb/core/kqp/compile/kqp_compile.cpp index 753873491dd..a2369af7d41 100644 --- a/ydb/core/kqp/compile/kqp_compile.cpp +++ b/ydb/core/kqp/compile/kqp_compile.cpp @@ -109,10 +109,14 @@ void FillColumns(const TCoAtomList& columns, const TKikimrTableMetadata& tableMe void FillLiteralKeyBound(const TCoDataCtor& literal, NKqpProto::TKqpPhyLiteralValue& proto) { auto type = literal.Ref().GetTypeAnn(); + + // TODO: support pg types + YQL_ENSURE(type->GetKind() != ETypeAnnotationKind::Pg, "pg types are not supported"); + auto slot = type->Cast<TDataExprType>()->GetSlot(); auto typeId = NKikimr::NUdf::GetDataTypeInfo(slot).TypeId; - YQL_ENSURE(NScheme::NTypeIds::IsYqlType(typeId) && NSchemeShard::IsAllowedKeyType(typeId)); + YQL_ENSURE(NScheme::NTypeIds::IsYqlType(typeId) && NSchemeShard::IsAllowedKeyType(NScheme::TTypeInfo(typeId))); auto& protoType = *proto.MutableType(); auto& protoValue = *proto.MutableValue(); diff --git a/ydb/core/kqp/compile/kqp_mkql_compiler.cpp b/ydb/core/kqp/compile/kqp_mkql_compiler.cpp index 04130446986..2786aca66d9 100644 --- a/ydb/core/kqp/compile/kqp_mkql_compiler.cpp +++ b/ydb/core/kqp/compile/kqp_mkql_compiler.cpp @@ -27,7 +27,8 @@ TVector<TKqpTableColumn> GetKqpColumns(const TKikimrTableMetadata& table, const auto columnData = table.Columns.FindPtr(name); if (columnData) { columnId = columnData->Id; - columnType = columnData->TypeId; + // TODO: support pg types + columnType = columnData->TypeInfo.GetTypeId(); notNull = columnData->NotNull; } else if (allowSystemColumns) { auto systemColumn = GetSystemColumns().find(name); @@ -90,9 +91,11 @@ void ValidateColumnsType(const TStreamExprType* streamType, const TKikimrTableMe for (auto* member : rowType->GetItems()) { auto columnData = tableMeta.Columns.FindPtr(member->GetName()); YQL_ENSURE(columnData); - YQL_ENSURE(columnData->TypeId); + auto columnDataType = columnData->TypeInfo.GetTypeId(); + // TODO: support pg types + YQL_ENSURE(columnDataType != 0 && columnDataType != NScheme::NTypeIds::Pg); - ValidateColumnType(member->GetItemType(), columnData->TypeId); + ValidateColumnType(member->GetItemType(), columnDataType); } } @@ -103,9 +106,11 @@ void ValidateRangeBoundType(const TTupleExprType* keyTupleType, const TKikimrTab for (ui32 i = 0; i < tableMeta.KeyColumnNames.size(); ++i) { auto columnData = tableMeta.Columns.FindPtr(tableMeta.KeyColumnNames[i]); YQL_ENSURE(columnData); - YQL_ENSURE(columnData->TypeId); + auto columnDataType = columnData->TypeInfo.GetTypeId(); + // TODO: support pg types + YQL_ENSURE(columnDataType != 0 && columnDataType != NScheme::NTypeIds::Pg); - ValidateColumnType(keyTupleType->GetItems()[i]->Cast<TOptionalExprType>()->GetItemType(), columnData->TypeId); + ValidateColumnType(keyTupleType->GetItems()[i]->Cast<TOptionalExprType>()->GetItemType(), columnDataType); } } @@ -143,12 +148,14 @@ TKqpKeyRange MakeKeyRange(const TKqlReadTableBase& readTable, const TKqlCompileC auto columnData = tableMeta.Columns.FindPtr(keyColumn); YQL_ENSURE(columnData); - YQL_ENSURE(columnData->TypeId); + auto columnDataType = columnData->TypeInfo.GetTypeId(); + // TODO: support pg types + YQL_ENSURE(columnDataType != 0 && columnDataType != NScheme::NTypeIds::Pg); - auto columnType = CreateColumnType(columnData->TypeId, ctx); + auto columnType = CreateColumnType(columnDataType, ctx); if (fromTuple.ArgCount() > i) { - ValidateColumnType(fromTuple.Arg(i).Ref().GetTypeAnn(), columnData->TypeId); + ValidateColumnType(fromTuple.Arg(i).Ref().GetTypeAnn(), columnDataType); fromValues.push_back(MkqlBuildExpr(fromTuple.Arg(i).Ref(), buildCtx)); } else if (fromInclusive) { fromValues.push_back(ctx.PgmBuilder().NewEmptyOptional( @@ -156,7 +163,7 @@ TKqpKeyRange MakeKeyRange(const TKqlReadTableBase& readTable, const TKqlCompileC } if (toTuple.ArgCount() > i) { - ValidateColumnType(toTuple.Arg(i).Ref().GetTypeAnn(), columnData->TypeId); + ValidateColumnType(toTuple.Arg(i).Ref().GetTypeAnn(), columnDataType); toValues.push_back(MkqlBuildExpr(toTuple.Arg(i).Ref(), buildCtx)); } else if (!toInclusive) { toValues.push_back(ctx.PgmBuilder().NewEmptyOptional( diff --git a/ydb/core/kqp/compute_actor/kqp_compute_state.cpp b/ydb/core/kqp/compute_actor/kqp_compute_state.cpp index 82d98eb152d..93e311b1c1e 100644 --- a/ydb/core/kqp/compute_actor/kqp_compute_state.cpp +++ b/ydb/core/kqp/compute_actor/kqp_compute_state.cpp @@ -21,7 +21,7 @@ void TCommonRetriesState::ResetRetry() { ResolveAttempt = 0; } -TString TShardState::PrintLastKey(TConstArrayRef<NScheme::TTypeId> keyTypes) const { +TString TShardState::PrintLastKey(TConstArrayRef<NScheme::TTypeInfo> keyTypes) const { if (LastKey.empty()) { return "<none>"; } @@ -41,7 +41,7 @@ TDuration TCommonRetriesState::CalcRetryDelay() { return LastRetryDelay; } -TString TShardState::ToString(TConstArrayRef<NScheme::TTypeId> keyTypes) const { +TString TShardState::ToString(TConstArrayRef<NScheme::TTypeInfo> keyTypes) const { TStringBuilder sb; sb << "TShardState{ TabletId: " << TabletId << ", State: " << State << ", Gen: " << Generation << ", Last Key " << TShardState::PrintLastKey(keyTypes) @@ -58,7 +58,7 @@ TString TShardState::ToString(TConstArrayRef<NScheme::TTypeId> keyTypes) const { return sb; } -const TSmallVec<TSerializedTableRange> TShardState::GetScanRanges(TConstArrayRef<NScheme::TTypeId> keyTypes) const { +const TSmallVec<TSerializedTableRange> TShardState::GetScanRanges(TConstArrayRef<NScheme::TTypeInfo> keyTypes) const { // No any data read previously, return all ranges if (!LastKey.DataSize()) { return Ranges; @@ -97,4 +97,4 @@ TString TShardState::GetAddress() const { sb << TabletId << "::" << ScannerIdx; return sb; } -}
\ No newline at end of file +} diff --git a/ydb/core/kqp/compute_actor/kqp_compute_state.h b/ydb/core/kqp/compute_actor/kqp_compute_state.h index 382d0709d9a..9fb389f53fa 100644 --- a/ydb/core/kqp/compute_actor/kqp_compute_state.h +++ b/ydb/core/kqp/compute_actor/kqp_compute_state.h @@ -83,15 +83,15 @@ struct TShardState: public TCommonRetriesState { TActorId ActorId; TOwnedCellVec LastKey; - TString PrintLastKey(TConstArrayRef<NScheme::TTypeId> keyTypes) const; + TString PrintLastKey(TConstArrayRef<NScheme::TTypeInfo> keyTypes) const; TShardState(const ui64 tabletId, const ui32 scanIdx) : TabletId(tabletId) , ScannerIdx(scanIdx) { } - TString ToString(TConstArrayRef<NScheme::TTypeId> keyTypes) const; - const TSmallVec<TSerializedTableRange> GetScanRanges(TConstArrayRef<NScheme::TTypeId> keyTypes) const; + TString ToString(TConstArrayRef<NScheme::TTypeInfo> keyTypes) const; + const TSmallVec<TSerializedTableRange> GetScanRanges(TConstArrayRef<NScheme::TTypeInfo> keyTypes) const; TString GetAddress() const; }; } diff --git a/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp b/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp index 68a4dd96fdf..b23b02f5d18 100644 --- a/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp +++ b/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp @@ -9,6 +9,7 @@ #include <ydb/core/kqp/runtime/kqp_channel_storage.h> #include <ydb/core/kqp/runtime/kqp_tasks_runner.h> #include <ydb/core/kqp/common/kqp_resolve.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/sys_view/scan.h> #include <ydb/core/tx/datashard/datashard_kqp_compute.h> #include <ydb/core/tx/datashard/datashard.h> @@ -132,7 +133,10 @@ public: YQL_ENSURE(!Meta.GetReads().empty()); YQL_ENSURE(Meta.GetTable().GetTableKind() != (ui32)ETableKind::SysView); - KeyColumnTypes.assign(Meta.GetKeyColumnTypes().begin(), Meta.GetKeyColumnTypes().end()); + KeyColumnTypes.reserve(Meta.GetKeyColumnTypes().size()); + for (auto typeId : Meta.GetKeyColumnTypes()) { + KeyColumnTypes.push_back(NScheme::TTypeInfo((NScheme::TTypeId)typeId)); + } } void DoBootstrap() { @@ -282,9 +286,15 @@ private: THolder<TEvDataShard::TEvKqpScan> BuildEvKqpScan(const ui32 scanId, const ui32 gen, const TSmallVec<TSerializedTableRange>& ranges) const { auto ev = MakeHolder<TEvDataShard::TEvKqpScan>(); ev->Record.SetLocalPathId(ScanData->TableId.PathId.LocalPathId); - for (auto& column : ScanData->GetColumns()) { + for (auto& column: ScanData->GetColumns()) { ev->Record.AddColumnTags(column.Tag); - ev->Record.AddColumnTypes(column.Type); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.Type); + ev->Record.AddColumnTypes(columnType.TypeId); + if (columnType.TypeInfo) { + *ev->Record.AddColumnTypeInfos() = *columnType.TypeInfo; + } else { + *ev->Record.AddColumnTypeInfos() = NKikimrProto::TTypeInfo(); + } } ev->Record.MutableSkipNullKeys()->CopyFrom(Meta.GetSkipNullKeys()); @@ -1204,7 +1214,7 @@ private: TIntrusivePtr<TKqpCounters> Counters; TScannedDataStats Stats; NKikimrTxDataShard::TKqpTransaction::TScanTaskMeta Meta; - TVector<NScheme::TTypeId> KeyColumnTypes; + TVector<NScheme::TTypeInfo> KeyColumnTypes; NMiniKQL::TKqpScanComputeContext::TScanData* ScanData = nullptr; std::deque<std::pair<TEvKqpCompute::TEvScanData::TPtr, TInstant>> PendingScanData; std::deque<TShardState> PendingShards; diff --git a/ydb/core/kqp/executer/kqp_data_executer.cpp b/ydb/core/kqp/executer/kqp_data_executer.cpp index 584524bdba9..ce8f5060d7c 100644 --- a/ydb/core/kqp/executer/kqp_data_executer.cpp +++ b/ydb/core/kqp/executer/kqp_data_executer.cpp @@ -1456,7 +1456,11 @@ private: for (auto& column : read.Columns) { auto* protoColumn = protoReadMeta->AddColumns(); protoColumn->SetId(column.Id); - protoColumn->SetType(column.Type); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.Type); + protoColumn->SetType(columnType.TypeId); + if (columnType.TypeInfo) { + *protoColumn->MutableTypeInfo() = *columnType.TypeInfo; + } protoColumn->SetName(column.Name); } protoReadMeta->SetItemsLimit(task.Meta.ReadInfo.ItemsLimit); @@ -1475,7 +1479,11 @@ private: auto& protoColumn = *protoColumnWrite.MutableColumn(); protoColumn.SetId(columnWrite.Column.Id); - protoColumn.SetType(columnWrite.Column.Type); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(columnWrite.Column.Type); + protoColumn.SetType(columnType.TypeId); + if (columnType.TypeInfo) { + *protoColumn.MutableTypeInfo() = *columnType.TypeInfo; + } protoColumn.SetName(columnWrite.Column.Name); protoColumnWrite.SetMaxValueSizeBytes(columnWrite.MaxValueSizeBytes); @@ -1494,7 +1502,11 @@ private: const auto& tableInfo = TableKeys.GetTable(stageInfo.Meta.TableId); for (const auto& keyColumnName : tableInfo.KeyColumns) { const auto& keyColumn = tableInfo.Columns.at(keyColumnName); - protoTaskMeta.AddKeyColumnTypes(keyColumn.Type); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(keyColumn.Type); + protoTaskMeta.AddKeyColumnTypes(columnType.TypeId); + if (columnType.TypeInfo) { + *protoTaskMeta.AddKeyColumnTypeInfos() = *columnType.TypeInfo; + } } for (bool skipNullKey : stageInfo.Meta.SkipNullKeys) { @@ -1507,7 +1519,11 @@ private: for (auto& column : task.Meta.Reads->front().Columns) { auto* protoColumn = protoTaskMeta.AddColumns(); protoColumn->SetId(column.Id); - protoColumn->SetType(column.Type); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.Type); + protoColumn->SetType(columnType.TypeId); + if (columnType.TypeInfo) { + *protoColumn->MutableTypeInfo() = *columnType.TypeInfo; + } protoColumn->SetName(column.Name); } @@ -1519,7 +1535,7 @@ private: YQL_ENSURE((int) read.Columns.size() == protoTaskMeta.GetColumns().size()); for (ui64 i = 0; i < read.Columns.size(); ++i) { YQL_ENSURE(read.Columns[i].Id == protoTaskMeta.GetColumns()[i].GetId()); - YQL_ENSURE(read.Columns[i].Type == protoTaskMeta.GetColumns()[i].GetType()); + YQL_ENSURE(read.Columns[i].Type.GetTypeId() == protoTaskMeta.GetColumns()[i].GetType()); } } diff --git a/ydb/core/kqp/executer/kqp_partition_helper.cpp b/ydb/core/kqp/executer/kqp_partition_helper.cpp index aa01c3da06c..1583ff2247a 100644 --- a/ydb/core/kqp/executer/kqp_partition_helper.cpp +++ b/ydb/core/kqp/executer/kqp_partition_helper.cpp @@ -2,6 +2,7 @@ #include "kqp_table_resolver.h" #include <ydb/core/base/appdata.h> +#include <ydb/core/kqp/common/kqp_types.h> #include <ydb/core/tx/datashard/range_ops.h> #include <ydb/library/mkql_proto/mkql_proto.h> @@ -122,8 +123,8 @@ THashMap<ui64, TShardParamValuesAndRanges> PartitionParamByKeyPrefix(const NDq:: const ui64 keyLen = table.KeyColumns.size(); - TVector<NUdf::TDataTypeId> keyFullType{Reserve(keyLen)}; - TVector<NUdf::TDataTypeId> keyPrefixType{Reserve(keyLen)}; + TVector<NScheme::TTypeInfo> keyFullType{Reserve(keyLen)}; + TVector<NScheme::TTypeInfo> keyPrefixType{Reserve(keyLen)}; TVector<ui32> keyPrefixIndices{Reserve(keyLen)}; for (const auto& keyColumn : table.KeyColumns) { @@ -132,8 +133,9 @@ THashMap<ui64, TShardParamValuesAndRanges> PartitionParamByKeyPrefix(const NDq:: break; } - keyFullType.push_back(columnInfo->GetTypeId()); - keyPrefixType.push_back(columnInfo->GetTypeId()); + auto typeInfo = NScheme::TypeInfoFromMiniKQLType(columnInfo->Type); + keyFullType.push_back(typeInfo); + keyPrefixType.push_back(typeInfo); keyPrefixIndices.push_back(columnInfo->Index); } @@ -187,7 +189,7 @@ THashMap<ui64, TShardParamValuesAndRanges> PartitionParamByKeyPrefix(const NDq:: return ret; } -TVector<TCell> FillKeyValues(const TVector<NUdf::TDataTypeId>& keyColumnTypes, const NKqpProto::TKqpPhyKeyBound& bound, +TVector<TCell> FillKeyValues(const TVector<NScheme::TTypeInfo>& keyColumnTypes, const NKqpProto::TKqpPhyKeyBound& bound, const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv) { YQL_ENSURE(bound.ValuesSize() <= keyColumnTypes.size()); @@ -243,7 +245,7 @@ TVector<TCell> FillKeyValues(const TVector<NUdf::TDataTypeId>& keyColumnTypes, c } TSerializedPointOrRange FillOneRange(NUdf::TUnboxedValue& begin, NUdf::TUnboxedValue& end, - const TVector<NUdf::TDataTypeId>& keyColumnTypes, const NMiniKQL::TTypeEnvironment& typeEnv) + const TVector<NScheme::TTypeInfo>& keyColumnTypes, const NMiniKQL::TTypeEnvironment& typeEnv) { const ui32 keyColumnsSize = keyColumnTypes.size(); @@ -345,7 +347,7 @@ TSerializedPointOrRange FillOneRange(NUdf::TUnboxedValue& begin, NUdf::TUnboxedV return range; } -TVector<TSerializedPointOrRange> BuildFullRange(const TVector<NUdf::TDataTypeId>& keyColumnTypes) { +TVector<TSerializedPointOrRange> BuildFullRange(const TVector<NScheme::TTypeInfo>& keyColumnTypes) { // Build range from NULL, NULL ... NULL to +inf, +inf ... +inf TVector<TCell> fromKeyValues(keyColumnTypes.size()); @@ -357,7 +359,7 @@ TVector<TSerializedPointOrRange> BuildFullRange(const TVector<NUdf::TDataTypeId> return {std::move(range)}; } -TVector<TSerializedPointOrRange> FillRangesFromParameter(const TVector<NUdf::TDataTypeId>& keyColumnTypes, +TVector<TSerializedPointOrRange> FillRangesFromParameter(const TVector<NScheme::TTypeInfo>& keyColumnTypes, const NKqpProto::TKqpPhyParamValue& rangesParam, const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv) { @@ -397,7 +399,7 @@ TVector<TSerializedPointOrRange> FillRangesFromParameter(const TVector<NUdf::TDa } template <typename PhyOpReadRanges> -TVector<TSerializedPointOrRange> FillReadRangesInternal(const TVector<NUdf::TDataTypeId>& keyColumnTypes, +TVector<TSerializedPointOrRange> FillReadRangesInternal(const TVector<NScheme::TTypeInfo>& keyColumnTypes, const PhyOpReadRanges& readRanges, const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv) { @@ -412,21 +414,21 @@ TVector<TSerializedPointOrRange> FillReadRangesInternal(const TVector<NUdf::TDat } // anonymous namespace -TVector<TSerializedPointOrRange> FillReadRanges(const TVector<NUdf::TDataTypeId>& keyColumnTypes, +TVector<TSerializedPointOrRange> FillReadRanges(const TVector<NScheme::TTypeInfo>& keyColumnTypes, const NKqpProto::TKqpPhyOpReadOlapRanges& readRange, const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv) { return FillReadRangesInternal(keyColumnTypes, readRange, stageInfo, holderFactory, typeEnv); } -TVector<TSerializedPointOrRange> FillReadRanges(const TVector<NUdf::TDataTypeId>& keyColumnTypes, +TVector<TSerializedPointOrRange> FillReadRanges(const TVector<NScheme::TTypeInfo>& keyColumnTypes, const NKqpProto::TKqpPhyOpReadRanges& readRange, const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv) { return FillReadRangesInternal(keyColumnTypes, readRange, stageInfo, holderFactory, typeEnv); } -TSerializedTableRange MakeKeyRange(const TVector<NUdf::TDataTypeId>& keyColumnTypes, +TSerializedTableRange MakeKeyRange(const TVector<NScheme::TTypeInfo>& keyColumnTypes, const NKqpProto::TKqpPhyKeyRange& range, const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv) { @@ -494,7 +496,7 @@ void FillFullRange(const TStageInfo& stageInfo, THashMap<ui64, TShardInfo>& shar } } // anonymous namespace -TString TShardInfo::ToString(const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const +TString TShardInfo::ToString(const TVector<NScheme::TTypeInfo>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const { TStringBuilder sb; sb << "TShardInfo{ "; diff --git a/ydb/core/kqp/executer/kqp_partition_helper.h b/ydb/core/kqp/executer/kqp_partition_helper.h index 06913939833..d9ad7e59c23 100644 --- a/ydb/core/kqp/executer/kqp_partition_helper.h +++ b/ydb/core/kqp/executer/kqp_partition_helper.h @@ -24,18 +24,18 @@ struct TShardInfo { TMaybe<TShardKeyRanges> KeyWriteRanges; // empty -> no writes THashMap<TString, TColumnWriteInfo> ColumnWrites; - TString ToString(const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const; + TString ToString(const TVector<NScheme::TTypeInfo>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const; }; -TSerializedTableRange MakeKeyRange(const TVector<NUdf::TDataTypeId>& keyColumnTypes, +TSerializedTableRange MakeKeyRange(const TVector<NScheme::TTypeInfo>& keyColumnTypes, const NKqpProto::TKqpPhyKeyRange& range, const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv); -TVector<TSerializedPointOrRange> FillReadRanges(const TVector<NUdf::TDataTypeId>& keyColumnTypes, +TVector<TSerializedPointOrRange> FillReadRanges(const TVector<NScheme::TTypeInfo>& keyColumnTypes, const NKqpProto::TKqpPhyOpReadOlapRanges& readRange, const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv); -TVector<TSerializedPointOrRange> FillReadRanges(const TVector<NUdf::TDataTypeId>& keyColumnTypes, +TVector<TSerializedPointOrRange> FillReadRanges(const TVector<NScheme::TTypeInfo>& keyColumnTypes, const NKqpProto::TKqpPhyOpReadRanges& readRange, const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv); diff --git a/ydb/core/kqp/executer/kqp_scan_executer.cpp b/ydb/core/kqp/executer/kqp_scan_executer.cpp index 45400e69f43..03739f79c0a 100644 --- a/ydb/core/kqp/executer/kqp_scan_executer.cpp +++ b/ydb/core/kqp/executer/kqp_scan_executer.cpp @@ -413,10 +413,11 @@ private: if (memberType->GetKind() == NKikimr::NMiniKQL::TType::EKind::Optional) { memberType = static_cast<NKikimr::NMiniKQL::TOptionalType*>(memberType)->GetItemType(); } + // TODO: support pg types YQL_ENSURE(memberType->GetKind() == NKikimr::NMiniKQL::TType::EKind::Data, "Expected simple data types to be read from column shard"); auto memberDataType = static_cast<NKikimr::NMiniKQL::TDataType*>(memberType); - taskMeta.ReadInfo.ResultColumnsTypes.push_back(memberDataType->GetSchemeType()); + taskMeta.ReadInfo.ResultColumnsTypes.push_back(NScheme::TTypeInfo(memberDataType->GetSchemeType())); } } @@ -734,7 +735,13 @@ private: const auto& tableInfo = TableKeys.GetTable(stageInfo.Meta.TableId); for (const auto& keyColumnName : tableInfo.KeyColumns) { const auto& keyColumn = tableInfo.Columns.at(keyColumnName); - protoTaskMeta.AddKeyColumnTypes(keyColumn.Type); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(keyColumn.Type); + protoTaskMeta.AddKeyColumnTypes(columnType.TypeId); + if (columnType.TypeInfo) { + *protoTaskMeta.AddKeyColumnTypeInfos() = *columnType.TypeInfo; + } else { + *protoTaskMeta.AddKeyColumnTypeInfos() = NKikimrProto::TTypeInfo(); + } } switch (tableInfo.TableKind) { @@ -772,7 +779,11 @@ private: for (auto columnType : task.Meta.ReadInfo.ResultColumnsTypes) { auto* protoResultColumn = protoTaskMeta.AddResultColumns(); protoResultColumn->SetId(0); - protoResultColumn->SetType(columnType); + auto protoColumnType = NScheme::ProtoColumnTypeFromTypeInfo(columnType); + protoResultColumn->SetType(protoColumnType.TypeId); + if (protoColumnType.TypeInfo) { + *protoResultColumn->MutableTypeInfo() = *protoColumnType.TypeInfo; + } } if (tableInfo.TableKind == ETableKind::Olap) { @@ -791,7 +802,11 @@ private: for (auto& column : task.Meta.Reads->front().Columns) { auto* protoColumn = protoTaskMeta.AddColumns(); protoColumn->SetId(column.Id); - protoColumn->SetType(column.Type); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.Type); + protoColumn->SetType(columnType.TypeId); + if (columnType.TypeInfo) { + *protoColumn->MutableTypeInfo() = *columnType.TypeInfo; + } protoColumn->SetName(column.Name); } } @@ -804,7 +819,7 @@ private: YQL_ENSURE((int) read.Columns.size() == protoTaskMeta.GetColumns().size()); for (ui64 i = 0; i < read.Columns.size(); ++i) { YQL_ENSURE(read.Columns[i].Id == protoTaskMeta.GetColumns()[i].GetId()); - YQL_ENSURE(read.Columns[i].Type == protoTaskMeta.GetColumns()[i].GetType()); + YQL_ENSURE(read.Columns[i].Type.GetTypeId() == protoTaskMeta.GetColumns()[i].GetType()); } nShardScans++; diff --git a/ydb/core/kqp/executer/kqp_table_resolver.cpp b/ydb/core/kqp/executer/kqp_table_resolver.cpp index c0f3f327b2c..d4b9d715aaf 100644 --- a/ydb/core/kqp/executer/kqp_table_resolver.cpp +++ b/ydb/core/kqp/executer/kqp_table_resolver.cpp @@ -123,7 +123,7 @@ private: for (auto& [columnName, columnKey] : table->Columns) { if (auto* systemColumn = sysColumns.FindPtr(columnName)) { columnKey.Id = systemColumn->ColumnId; - columnKey.Type = systemColumn->TypeId; + columnKey.Type = NScheme::TTypeInfo(systemColumn->TypeId); continue; } diff --git a/ydb/core/kqp/executer/kqp_tasks_graph.cpp b/ydb/core/kqp/executer/kqp_tasks_graph.cpp index 08dd6b985f7..cdab73ccfec 100644 --- a/ydb/core/kqp/executer/kqp_tasks_graph.cpp +++ b/ydb/core/kqp/executer/kqp_tasks_graph.cpp @@ -409,7 +409,7 @@ void TShardKeyRanges::MakeFull(TSerializedPointOrRange&& pointOrRange) { } -void TShardKeyRanges::MergeWritePoints(TShardKeyRanges&& other, const TVector<NScheme::TTypeId>& keyTypes) { +void TShardKeyRanges::MergeWritePoints(TShardKeyRanges&& other, const TVector<NScheme::TTypeInfo>& keyTypes) { #ifdef DBG_TRACE Cerr << (TStringBuilder() << "-- merge " << ToString(keyTypes, *AppData()->TypeRegistry) << " with " << other.ToString(keyTypes, *AppData()->TypeRegistry) << Endl); @@ -554,7 +554,7 @@ void TShardKeyRanges::MergeWritePoints(TShardKeyRanges&& other, const TVector<NS #endif } -TString TShardKeyRanges::ToString(const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const +TString TShardKeyRanges::ToString(const TVector<NScheme::TTypeInfo>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const { TStringBuilder sb; sb << "TShardKeyRanges{ "; @@ -633,7 +633,7 @@ std::pair<const TSerializedCellVec*, bool> TShardKeyRanges::GetRightBorder() con return !lastRange.Point ? std::make_pair(&lastRange.To, lastRange.ToInclusive) : std::make_pair(&lastRange.From, true); } -TString TTaskMeta::ToString(const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const +TString TTaskMeta::ToString(const TVector<NScheme::TTypeInfo>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const { TStringBuilder sb; sb << "TTaskMeta{ ShardId: " << ShardId << ", Params: ["; diff --git a/ydb/core/kqp/executer/kqp_tasks_graph.h b/ydb/core/kqp/executer/kqp_tasks_graph.h index 51da52a204d..a7d718f9254 100644 --- a/ydb/core/kqp/executer/kqp_tasks_graph.h +++ b/ydb/core/kqp/executer/kqp_tasks_graph.h @@ -86,9 +86,9 @@ struct TShardKeyRanges { bool IsFullRange() const { return FullRange.has_value(); } TVector<TSerializedPointOrRange>& GetRanges() { return Ranges; } - void MergeWritePoints(TShardKeyRanges&& other, const TVector<NScheme::TTypeId>& keyTypes); + void MergeWritePoints(TShardKeyRanges&& other, const TVector<NScheme::TTypeInfo>& keyTypes); - TString ToString(const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const; + TString ToString(const TVector<NScheme::TTypeInfo>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const; void SerializeTo(NKikimrTxDataShard::TKqpTransaction_TDataTaskMeta_TKeyRange* proto) const; void SerializeTo(NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta_TReadOpMeta* proto) const; @@ -105,7 +105,7 @@ struct TTaskMeta { struct TColumn { ui32 Id = 0; - ui32 Type = 0; + NScheme::TTypeInfo Type; TString Name; }; @@ -130,7 +130,7 @@ struct TTaskMeta { bool Reverse = false; bool Sorted = false; TKqpOlapProgram OlapProgram; - TVector<NUdf::TDataTypeId> ResultColumnsTypes; + TVector<NScheme::TTypeInfo> ResultColumnsTypes; }; struct TWriteInfo { @@ -157,7 +157,7 @@ struct TTaskMeta { TMaybe<TVector<TShardReadInfo>> Reads; // if not set -> no reads TMaybe<TWriteInfo> Writes; // if not set -> no writes - TString ToString(const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const; + TString ToString(const TVector<NScheme::TTypeInfo>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) const; }; using TStageInfo = NYql::NDq::TStageInfo<TStageInfoMeta>; diff --git a/ydb/core/kqp/executer/ut/CMakeLists.darwin.txt b/ydb/core/kqp/executer/ut/CMakeLists.darwin.txt index 6bc4f140024..c07898ed2eb 100644 --- a/ydb/core/kqp/executer/ut/CMakeLists.darwin.txt +++ b/ydb/core/kqp/executer/ut/CMakeLists.darwin.txt @@ -24,6 +24,7 @@ target_link_libraries(ydb-core-kqp-executer-ut PUBLIC core-kqp-host kqp-ut-common cpp-client-ydb_proto + yql-sql-pg_dummy ) target_link_options(ydb-core-kqp-executer-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/kqp/executer/ut/CMakeLists.linux.txt b/ydb/core/kqp/executer/ut/CMakeLists.linux.txt index 2847c2460a2..3ace8e1805d 100644 --- a/ydb/core/kqp/executer/ut/CMakeLists.linux.txt +++ b/ydb/core/kqp/executer/ut/CMakeLists.linux.txt @@ -26,6 +26,7 @@ target_link_libraries(ydb-core-kqp-executer-ut PUBLIC core-kqp-host kqp-ut-common cpp-client-ydb_proto + yql-sql-pg_dummy ) target_link_options(ydb-core-kqp-executer-ut PRIVATE -ldl diff --git a/ydb/core/kqp/kqp_metadata_loader.cpp b/ydb/core/kqp/kqp_metadata_loader.cpp index 79d5c5b7715..eab53a449c3 100644 --- a/ydb/core/kqp/kqp_metadata_loader.cpp +++ b/ydb/core/kqp/kqp_metadata_loader.cpp @@ -151,7 +151,9 @@ TTableMetadataResult GetLoadTableMetadataResult(const NSchemeCache::TSchemeCache const auto& columnDesc = pair.second; TString typeName; - YQL_ENSURE(NScheme::TryGetTypeName(columnDesc.PType, typeName)); + // TODO: support pg types + YQL_ENSURE(columnDesc.PType.GetTypeId() != NScheme::NTypeIds::Pg); + YQL_ENSURE(NScheme::TryGetTypeName(columnDesc.PType.GetTypeId(), typeName)); auto notNull = entry.NotNullColumns.contains(columnDesc.Name); tableMeta->Columns.emplace(columnDesc.Name, NYql::TKikimrColumnMetadata( columnDesc.Name, columnDesc.Id, typeName, notNull, columnDesc.PType)); diff --git a/ydb/core/kqp/node/ut/CMakeLists.darwin.txt b/ydb/core/kqp/node/ut/CMakeLists.darwin.txt index 2d1867e1874..959894d5912 100644 --- a/ydb/core/kqp/node/ut/CMakeLists.darwin.txt +++ b/ydb/core/kqp/node/ut/CMakeLists.darwin.txt @@ -21,6 +21,7 @@ target_link_libraries(ydb-core-kqp-node-ut PUBLIC cpp-testing-unittest_main core-kqp-node kqp-ut-common + yql-sql-pg_dummy ) target_link_options(ydb-core-kqp-node-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/kqp/node/ut/CMakeLists.linux.txt b/ydb/core/kqp/node/ut/CMakeLists.linux.txt index ca29c999a9a..7af039a1c5a 100644 --- a/ydb/core/kqp/node/ut/CMakeLists.linux.txt +++ b/ydb/core/kqp/node/ut/CMakeLists.linux.txt @@ -23,6 +23,7 @@ target_link_libraries(ydb-core-kqp-node-ut PUBLIC cpp-testing-unittest_main core-kqp-node kqp-ut-common + yql-sql-pg_dummy ) target_link_options(ydb-core-kqp-node-ut PRIVATE -ldl diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp index a97c65bedbc..88f103fa05c 100644 --- a/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp +++ b/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp @@ -348,9 +348,11 @@ bool RequireLookupPrecomputeStage(const TKqlLookupTable& lookup) { if (tuple.Value().Maybe<TCoParameter>()) { // pass } else if (tuple.Value().Maybe<TCoDataCtor>()) { + // TODO: support pg types auto slot = tuple.Value().Ref().GetTypeAnn()->Cast<TDataExprType>()->GetSlot(); auto typeId = NUdf::GetDataTypeInfo(slot).TypeId; - if (NScheme::NTypeIds::IsYqlType(typeId) && NSchemeShard::IsAllowedKeyType(typeId)) { + auto typeInfo = NScheme::TTypeInfo(typeId); + if (NScheme::NTypeIds::IsYqlType(typeId) && NSchemeShard::IsAllowedKeyType(typeInfo)) { // pass } else { return true; diff --git a/ydb/core/kqp/provider/ut/CMakeLists.darwin.txt b/ydb/core/kqp/provider/ut/CMakeLists.darwin.txt index e65803fd5a0..49a6a686d1b 100644 --- a/ydb/core/kqp/provider/ut/CMakeLists.darwin.txt +++ b/ydb/core/kqp/provider/ut/CMakeLists.darwin.txt @@ -22,6 +22,7 @@ target_link_libraries(ydb-core-kqp-provider-ut PUBLIC core-kqp-provider core-client-minikql_result_lib kqp-ut-common + yql-sql-pg_dummy ) target_link_options(ydb-core-kqp-provider-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/kqp/provider/ut/CMakeLists.linux.txt b/ydb/core/kqp/provider/ut/CMakeLists.linux.txt index c1bba850449..6bae32f1fd3 100644 --- a/ydb/core/kqp/provider/ut/CMakeLists.linux.txt +++ b/ydb/core/kqp/provider/ut/CMakeLists.linux.txt @@ -24,6 +24,7 @@ target_link_libraries(ydb-core-kqp-provider-ut PUBLIC core-kqp-provider core-client-minikql_result_lib kqp-ut-common + yql-sql-pg_dummy ) target_link_options(ydb-core-kqp-provider-ut PRIVATE -ldl diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp index e49f84b5e1e..24e9df6fbec 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp @@ -2,6 +2,7 @@ #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <ydb/library/yql/providers/common/proto/gateways_config.pb.h> +#include <ydb/library/yql/utils/pg_types.h> #include <ydb/library/yql/utils/yql_panic.h> #include <ydb/core/base/table_index.h> @@ -274,6 +275,14 @@ EYqlIssueCode YqlStatusFromYdbStatus(ui32 ydbStatus) { } void SetColumnType(Ydb::Type& protoType, const TString& typeName, bool notNull) { + auto* typeDesc = NKikimr::NPg::TypeDescFromPgTypeName(typeName); + if (typeDesc) { + auto pg = notNull ? protoType.mutable_pg_type() : + protoType.mutable_optional_type()->mutable_item()->mutable_pg_type(); + pg->set_oid(NKikimr::NPg::PgTypeIdFromTypeDesc(typeDesc)); + return; + } + NUdf::EDataSlot dataSlot = NUdf::GetDataSlot(typeName); if (dataSlot == NUdf::EDataSlot::Decimal) { auto decimal = notNull ? protoType.mutable_decimal_type() : diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index 7598494a2b9..7f593978320 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -9,6 +9,7 @@ #include <ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h> // TODO: Switch to public protobufs +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/protos/kqp.pb.h> #include <ydb/library/yql/dq/actors/protos/dq_events.pb.h> #include <ydb/core/protos/flat_scheme_op.pb.h> @@ -182,17 +183,17 @@ struct TKikimrColumnMetadata { ui32 Id = 0; TString Type; bool NotNull = false; - ui32 TypeId = 0; + NKikimr::NScheme::TTypeInfo TypeInfo; TVector<TString> Families; TKikimrColumnMetadata() = default; - TKikimrColumnMetadata(const TString& name, ui32 id, const TString& type, bool notNull, ui32 typeId = 0) + TKikimrColumnMetadata(const TString& name, ui32 id, const TString& type, bool notNull, NKikimr::NScheme::TTypeInfo typeInfo = {}) : Name(name) , Id(id) , Type(type) , NotNull(notNull) - , TypeId(typeId) + , TypeInfo(typeInfo) {} explicit TKikimrColumnMetadata(const NKikimrKqp::TKqpColumnMetadataProto* message) @@ -200,16 +201,22 @@ struct TKikimrColumnMetadata { , Id(message->GetId()) , Type(message->GetType()) , NotNull(message->GetNotNull()) - , TypeId(message->GetTypeId()) , Families(message->GetFamily().begin(), message->GetFamily().end()) - {} + { + TypeInfo = NKikimr::NScheme::TypeInfoFromProtoColumnType(message->GetTypeId(), + message->HasTypeInfo() ? &message->GetTypeInfo() : nullptr); + } void ToMessage(NKikimrKqp::TKqpColumnMetadataProto* message) const { message->SetName(Name); message->SetId(Id); message->SetType(Type); message->SetNotNull(NotNull); - message->SetTypeId(TypeId); + auto columnType = NKikimr::NScheme::ProtoColumnTypeFromTypeInfo(TypeInfo); + message->SetTypeId(columnType.TypeId); + if (columnType.TypeInfo) { + *message->MutableTypeInfo() = *columnType.TypeInfo; + } for(auto& family: Families) { message->AddFamily(family); } diff --git a/ydb/core/kqp/proxy/ut/CMakeLists.darwin.txt b/ydb/core/kqp/proxy/ut/CMakeLists.darwin.txt index 7cf2b981ee7..13fed208c6f 100644 --- a/ydb/core/kqp/proxy/ut/CMakeLists.darwin.txt +++ b/ydb/core/kqp/proxy/ut/CMakeLists.darwin.txt @@ -21,6 +21,7 @@ target_link_libraries(ydb-core-kqp-proxy-ut PUBLIC cpp-testing-unittest_main core-kqp-proxy kqp-ut-common + yql-sql-pg_dummy ) target_link_options(ydb-core-kqp-proxy-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/kqp/proxy/ut/CMakeLists.linux.txt b/ydb/core/kqp/proxy/ut/CMakeLists.linux.txt index 757cb07b9d2..83446b90336 100644 --- a/ydb/core/kqp/proxy/ut/CMakeLists.linux.txt +++ b/ydb/core/kqp/proxy/ut/CMakeLists.linux.txt @@ -23,6 +23,7 @@ target_link_libraries(ydb-core-kqp-proxy-ut PUBLIC cpp-testing-unittest_main core-kqp-proxy kqp-ut-common + yql-sql-pg_dummy ) target_link_options(ydb-core-kqp-proxy-ut PRIVATE -ldl diff --git a/ydb/core/kqp/rm/ut/CMakeLists.darwin.txt b/ydb/core/kqp/rm/ut/CMakeLists.darwin.txt index bf1a22ecb18..636534daa8c 100644 --- a/ydb/core/kqp/rm/ut/CMakeLists.darwin.txt +++ b/ydb/core/kqp/rm/ut/CMakeLists.darwin.txt @@ -21,6 +21,7 @@ target_link_libraries(ydb-core-kqp-rm-ut PUBLIC cpp-testing-unittest_main core-kqp-rm kqp-ut-common + yql-sql-pg_dummy ) target_link_options(ydb-core-kqp-rm-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/kqp/rm/ut/CMakeLists.linux.txt b/ydb/core/kqp/rm/ut/CMakeLists.linux.txt index f7aa1aec538..f90487829b6 100644 --- a/ydb/core/kqp/rm/ut/CMakeLists.linux.txt +++ b/ydb/core/kqp/rm/ut/CMakeLists.linux.txt @@ -23,6 +23,7 @@ target_link_libraries(ydb-core-kqp-rm-ut PUBLIC cpp-testing-unittest_main core-kqp-rm kqp-ut-common + yql-sql-pg_dummy ) target_link_options(ydb-core-kqp-rm-ut PRIVATE -ldl diff --git a/ydb/core/kqp/runtime/kqp_compute.h b/ydb/core/kqp/runtime/kqp_compute.h index c55a4e2a21b..ae90fd8e466 100644 --- a/ydb/core/kqp/runtime/kqp_compute.h +++ b/ydb/core/kqp/runtime/kqp_compute.h @@ -14,7 +14,7 @@ class TKqpComputeContextBase : public NYql::NDq::TDqComputeContextBase { public: struct TColumn { NTable::TTag Tag; - NScheme::TTypeId Type; + NScheme::TTypeInfo Type; }; // used only at then building of a computation graph, to inject taskId in runtime nodes diff --git a/ydb/core/kqp/runtime/kqp_output_stream.cpp b/ydb/core/kqp/runtime/kqp_output_stream.cpp index b3aac31aaed..b931881f785 100644 --- a/ydb/core/kqp/runtime/kqp_output_stream.cpp +++ b/ydb/core/kqp/runtime/kqp_output_stream.cpp @@ -21,7 +21,7 @@ class TKqpOutputRangePartitionConsumer : public IDqOutputConsumer { public: TKqpOutputRangePartitionConsumer(const TTypeEnvironment& typeEnv, TVector<NYql::NDq::IDqOutput::TPtr>&& outputs, TVector<TKqpRangePartition>&& partitions, - TVector<TDataTypeId>&& keyColumnTypes, TVector<ui32>&& keyColumnIndices) + TVector<NScheme::TTypeInfo>&& keyColumnTypes, TVector<ui32>&& keyColumnIndices) : TypeEnv(typeEnv) , Outputs(std::move(outputs)) , Partitions(std::move(partitions)) @@ -59,7 +59,7 @@ private: const TTypeEnvironment& TypeEnv; TVector<NYql::NDq::IDqOutput::TPtr> Outputs; TVector<TKqpRangePartition> Partitions; - TVector<TDataTypeId> KeyColumnTypes; + TVector<NScheme::TTypeInfo> KeyColumnTypes; TVector<ui32> KeyColumnIndices; }; @@ -67,7 +67,7 @@ private: NYql::NDq::IDqOutputConsumer::TPtr CreateOutputRangePartitionConsumer( TVector<NYql::NDq::IDqOutput::TPtr>&& outputs, TVector<TKqpRangePartition>&& partitions, - TVector<NUdf::TDataTypeId>&& keyColumnTypes, TVector<ui32>&& keyColumnIndices, + TVector<NScheme::TTypeInfo>&& keyColumnTypes, TVector<ui32>&& keyColumnIndices, const NMiniKQL::TTypeEnvironment& typeEnv) { return MakeIntrusive<TKqpOutputRangePartitionConsumer>(typeEnv, std::move(outputs), std::move(partitions), diff --git a/ydb/core/kqp/runtime/kqp_read_table.cpp b/ydb/core/kqp/runtime/kqp_read_table.cpp index efbfb2a8087..9710d154ff2 100644 --- a/ydb/core/kqp/runtime/kqp_read_table.cpp +++ b/ydb/core/kqp/runtime/kqp_read_table.cpp @@ -5,6 +5,8 @@ #include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/core/kqp/common/kqp_types.h> + namespace NKikimr { namespace NKqp { @@ -52,7 +54,8 @@ static TCell BuildKeyTupleCell(const TType* type, const TUnboxedValue& value, co keyValue = value.GetOptionalValue(); } - return MakeCell(AS_TYPE(TDataType, keyType)->GetSchemeType(), keyValue, env, true); + auto typeInfo = NScheme::TypeInfoFromMiniKQLType(keyType); + return MakeCell(typeInfo, keyValue, env, true); } void BuildKeyTupleCells(const TTupleType* tupleType, const TUnboxedValue& tupleValue, TVector<TCell>& cells, @@ -72,7 +75,7 @@ void BuildKeyTupleCells(const TTupleType* tupleType, const TUnboxedValue& tupleV void ParseReadColumns(const TType* readType, const TRuntimeNode& tagsNode, TSmallVec<NTable::TTag>& columns, TSmallVec<NTable::TTag>& systemColumns) { - MKQL_ENSURE_S(readType); + MKQL_ENSURE_S(readType); MKQL_ENSURE_S(readType->GetKind() == TType::EKind::Flow); auto tags = AS_VALUE(TStructLiteral, tagsNode); diff --git a/ydb/core/kqp/runtime/kqp_runtime_impl.h b/ydb/core/kqp/runtime/kqp_runtime_impl.h index 004af2a5d6d..d1c504ef7c5 100644 --- a/ydb/core/kqp/runtime/kqp_runtime_impl.h +++ b/ydb/core/kqp/runtime/kqp_runtime_impl.h @@ -19,7 +19,7 @@ NUdf::TDataTypeId UnwrapDataTypeFromStruct(const NMiniKQL::TStructType& structTy NYql::NDq::IDqOutputConsumer::TPtr CreateOutputRangePartitionConsumer( TVector<NYql::NDq::IDqOutput::TPtr>&& outputs, TVector<TKqpRangePartition>&& partitions, - TVector<NUdf::TDataTypeId>&& keyColumnTypes, TVector<ui32>&& keyColumnIndices, + TVector<NScheme::TTypeInfo>&& keyColumnTypes, TVector<ui32>&& keyColumnIndices, const NMiniKQL::TTypeEnvironment& typeEnv); NYql::NDq::IDqOutputConsumer::TPtr CreateKqpApplyEffectsConsumer(NUdf::IApplyContext* applyCtx); diff --git a/ydb/core/kqp/runtime/kqp_scan_data.cpp b/ydb/core/kqp/runtime/kqp_scan_data.cpp index d4e74b69c0a..2c39d685db5 100644 --- a/ydb/core/kqp/runtime/kqp_scan_data.cpp +++ b/ydb/core/kqp/runtime/kqp_scan_data.cpp @@ -2,6 +2,7 @@ #include <ydb/core/engine/minikql/minikql_engine_host.h> #include <ydb/core/protos/tx_datashard.pb.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/library/yql/minikql/mkql_string_util.h> #include <ydb/library/yql/utils/yql_panic.h> @@ -22,12 +23,12 @@ struct TBytesStatistics { }; -TBytesStatistics GetUnboxedValueSize(const NUdf::TUnboxedValue& value, NScheme::TTypeId type) { +TBytesStatistics GetUnboxedValueSize(const NUdf::TUnboxedValue& value, NScheme::TTypeInfo type) { namespace NTypeIds = NScheme::NTypeIds; if (!value) { return {sizeof(NUdf::TUnboxedValue), 8}; // Special value for NULL elements } - switch (type) { + switch (type.GetTypeId()) { case NTypeIds::Bool: case NTypeIds::Int8: case NTypeIds::Uint8: @@ -48,12 +49,12 @@ TBytesStatistics GetUnboxedValueSize(const NUdf::TUnboxedValue& value, NScheme:: case NTypeIds::Interval: case NTypeIds::ActorId: case NTypeIds::StepOrderId: { - YQL_ENSURE(value.IsEmbedded(), "Passed wrong type: " << NScheme::TypeName(type)); + YQL_ENSURE(value.IsEmbedded(), "Passed wrong type: " << NScheme::TypeName(type.GetTypeId())); return {sizeof(NUdf::TUnboxedValue), sizeof(i64)}; } case NTypeIds::Decimal: { - YQL_ENSURE(value.IsEmbedded(), "Passed wrong type: " << NScheme::TypeName(type)); + YQL_ENSURE(value.IsEmbedded(), "Passed wrong type: " << NScheme::TypeName(type.GetTypeId())); return {sizeof(NUdf::TUnboxedValue), sizeof(NYql::NDecimal::TInt128)}; } case NTypeIds::String: @@ -71,8 +72,12 @@ TBytesStatistics GetUnboxedValueSize(const NUdf::TUnboxedValue& value, NScheme:: } } + case NTypeIds::Pg: + // TODO: support pg types + YQL_ENSURE(false, "pg types are not supported"); + default: - Y_VERIFY_DEBUG_S(false, "Unsupported type " << NScheme::TypeName(type)); + Y_VERIFY_DEBUG_S(false, "Unsupported type " << NScheme::TypeName(type.GetTypeId())); if (value.IsEmbedded()) { return {sizeof(NUdf::TUnboxedValue), sizeof(NUdf::TUnboxedValue)}; } else { @@ -139,7 +144,7 @@ NUdf::TUnboxedValue MakeUnboxedValueFromDecimal128Array(arrow::Array* column, ui } TBytesStatistics WriteColumnValuesFromArrow(const TVector<NUdf::TUnboxedValue*>& editAccessors, - const arrow::RecordBatch& batch, i64 columnIndex, NScheme::TTypeId columnType) + const arrow::RecordBatch& batch, i64 columnIndex, NScheme::TTypeInfo columnType) { TBytesStatistics columnStats; // Hold pointer to column until function end @@ -151,7 +156,7 @@ TBytesStatistics WriteColumnValuesFromArrow(const TVector<NUdf::TUnboxedValue*>& if (columnPtr->IsNull(rowIndex)) { rowItem = NUdf::TUnboxedValue(); } else { - switch(columnType) { + switch(columnType.GetTypeId()) { case NTypeIds::Bool: { rowItem = MakeUnboxedValue<arrow::BooleanArray, bool>(columnPtr, rowIndex); break; @@ -228,12 +233,16 @@ TBytesStatistics WriteColumnValuesFromArrow(const TVector<NUdf::TUnboxedValue*>& case NTypeIds::PairUi64Ui64: case NTypeIds::ActorId: case NTypeIds::StepOrderId: { - Y_VERIFY_DEBUG_S(false, "Unsupported (deprecated) type: " << NScheme::TypeName(columnType)); + Y_VERIFY_DEBUG_S(false, "Unsupported (deprecated) type: " << NScheme::TypeName(columnType.GetTypeId())); rowItem = MakeUnboxedValueFromFixedSizeBinaryData(columnPtr, rowIndex); break; } + case NTypeIds::Pg: + // TODO: support pg types + YQL_ENSURE(false, "Unsupported pg type at column " << columnIndex); + default: - YQL_ENSURE(false, "Unsupported type: " << NScheme::TypeName(columnType) << " at column " << columnIndex); + YQL_ENSURE(false, "Unsupported type: " << NScheme::TypeName(columnType.GetTypeId()) << " at column " << columnIndex); } } columnStats.AddStatistics(GetUnboxedValueSize(rowItem, columnType)); @@ -243,7 +252,7 @@ TBytesStatistics WriteColumnValuesFromArrow(const TVector<NUdf::TUnboxedValue*>& } // namespace -std::pair<ui64, ui64> GetUnboxedValueSizeForTests(const NUdf::TUnboxedValue& value, NScheme::TTypeId type) { +std::pair<ui64, ui64> GetUnboxedValueSizeForTests(const NUdf::TUnboxedValue& value, NScheme::TTypeInfo type) { auto sizes = GetUnboxedValueSize(value, type); return {sizes.AllocatedBytes, sizes.DataBytes}; } @@ -273,7 +282,8 @@ TKqpScanComputeContext::TScanData::TScanData(const NKikimrTxDataShard::TKqpTrans for (const auto& column : meta.GetColumns()) { NMiniKQL::TKqpScanComputeContext::TColumn c; c.Tag = column.GetId(); - c.Type = column.GetType(); + c.Type = NScheme::TypeInfoFromProtoColumnType(column.GetType(), + column.HasTypeInfo() ? &column.GetTypeInfo() : nullptr); if (!IsSystemColumn(c.Tag)) { Columns.emplace_back(std::move(c)); @@ -289,7 +299,8 @@ TKqpScanComputeContext::TScanData::TScanData(const NKikimrTxDataShard::TKqpTrans for (const auto& resColumn : meta.GetResultColumns()) { NMiniKQL::TKqpScanComputeContext::TColumn c; c.Tag = resColumn.GetId(); - c.Type = resColumn.GetType(); + c.Type = NScheme::TypeInfoFromProtoColumnType(resColumn.GetType(), + resColumn.HasTypeInfo() ? &resColumn.GetTypeInfo() : nullptr); if (!IsSystemColumn(c.Tag)) { ResultColumns.emplace_back(std::move(c)); diff --git a/ydb/core/kqp/runtime/kqp_scan_data.h b/ydb/core/kqp/runtime/kqp_scan_data.h index 0b052f7a151..e365ffa0933 100644 --- a/ydb/core/kqp/runtime/kqp_scan_data.h +++ b/ydb/core/kqp/runtime/kqp_scan_data.h @@ -22,7 +22,7 @@ namespace NKikimrTxDataShard { namespace NKikimr { namespace NMiniKQL { -std::pair<ui64, ui64> GetUnboxedValueSizeForTests(const NUdf::TUnboxedValue& value, NScheme::TTypeId type); +std::pair<ui64, ui64> GetUnboxedValueSizeForTests(const NUdf::TUnboxedValue& value, NScheme::TTypeInfo type); class IKqpTableReader : public TSimpleRefCount<IKqpTableReader> { public: diff --git a/ydb/core/kqp/runtime/kqp_scan_data_ut.cpp b/ydb/core/kqp/runtime/kqp_scan_data_ut.cpp index 43b7c44bf26..8f977f0f198 100644 --- a/ydb/core/kqp/runtime/kqp_scan_data_ut.cpp +++ b/ydb/core/kqp/runtime/kqp_scan_data_ut.cpp @@ -9,30 +9,31 @@ namespace NKikimr::NMiniKQL { namespace { namespace NTypeIds = NScheme::NTypeIds; using TTypeId = NScheme::TTypeId; +using TTypeInfo = NScheme::TTypeInfo; struct TDataRow { TSmallVec<TKqpComputeContextBase::TColumn> Columns() { return { - {0, NTypeIds::Bool}, - {1, NTypeIds::Int8}, - {2, NTypeIds::Int16}, - {3, NTypeIds::Int32}, - {4, NTypeIds::Int64}, - {5, NTypeIds::Uint8}, - {6, NTypeIds::Uint16}, - {7, NTypeIds::Uint32}, - {8, NTypeIds::Uint64}, - {9, NTypeIds::Float}, - {10, NTypeIds::Double}, - {11, NTypeIds::String}, - {12, NTypeIds::Utf8}, - {13, NTypeIds::Json}, - {14, NTypeIds::Yson}, - {15, NTypeIds::Date}, - {16, NTypeIds::Datetime}, - {17, NTypeIds::Timestamp}, - {18, NTypeIds::Interval}, - {19, NTypeIds::Decimal}, + {0, TTypeInfo(NTypeIds::Bool)}, + {1, TTypeInfo(NTypeIds::Int8)}, + {2, TTypeInfo(NTypeIds::Int16)}, + {3, TTypeInfo(NTypeIds::Int32)}, + {4, TTypeInfo(NTypeIds::Int64)}, + {5, TTypeInfo(NTypeIds::Uint8)}, + {6, TTypeInfo(NTypeIds::Uint16)}, + {7, TTypeInfo(NTypeIds::Uint32)}, + {8, TTypeInfo(NTypeIds::Uint64)}, + {9, TTypeInfo(NTypeIds::Float)}, + {10, TTypeInfo(NTypeIds::Double)}, + {11, TTypeInfo(NTypeIds::String)}, + {12, TTypeInfo(NTypeIds::Utf8)}, + {13, TTypeInfo(NTypeIds::Json)}, + {14, TTypeInfo(NTypeIds::Yson)}, + {15, TTypeInfo(NTypeIds::Date)}, + {16, TTypeInfo(NTypeIds::Datetime)}, + {17, TTypeInfo(NTypeIds::Timestamp)}, + {18, TTypeInfo(NTypeIds::Interval)}, + {19, TTypeInfo(NTypeIds::Decimal)}, }; } @@ -234,7 +235,7 @@ Y_UNIT_TEST_SUITE(TKqpScanData) { }; for (auto& testCase: cases) { - auto sizes = GetUnboxedValueSizeForTests(testCase.Value, testCase.Type); + auto sizes = GetUnboxedValueSizeForTests(testCase.Value, NScheme::TTypeInfo(testCase.Type)); UNIT_ASSERT_EQUAL_C(sizes, testCase.ExpectedSizes, "Wrong size for type " << NScheme::TypeName(testCase.Type)); } } @@ -293,7 +294,7 @@ Y_UNIT_TEST_SUITE(TKqpScanData) { TSmallVec<TKqpComputeContextBase::TColumn> resultCols; auto resCol = TKqpComputeContextBase::TColumn { - .Type = NTypeIds::Int8 + .Type = TTypeInfo(NTypeIds::Int8) }; resultCols.push_back(resCol); TKqpScanComputeContext::TScanData scanData({}, TTableRange({}), rows.front().Columns(), {}, {}, resultCols); diff --git a/ydb/core/kqp/runtime/kqp_stream_lookup_actor.cpp b/ydb/core/kqp/runtime/kqp_stream_lookup_actor.cpp index 103c72098bb..b440bd6e969 100644 --- a/ydb/core/kqp/runtime/kqp_stream_lookup_actor.cpp +++ b/ydb/core/kqp/runtime/kqp_stream_lookup_actor.cpp @@ -116,7 +116,7 @@ private: struct TTableScheme { TTableScheme(const THashMap<ui32, TSysTables::TTableColumnInfo>& columns) { - std::map<ui32, NKikimr::NScheme::TTypeId> keyColumnTypesByKeyOrder; + std::map<ui32, NScheme::TTypeInfo> keyColumnTypesByKeyOrder; for (const auto& [_, column] : columns) { if (column.KeyOrder >= 0) { keyColumnTypesByKeyOrder[column.KeyOrder] = column.PType; @@ -132,7 +132,7 @@ private: } std::unordered_map<TString, TSysTables::TTableColumnInfo> ColumnsByName; - std::vector<NKikimr::NScheme::TTypeId> KeyColumnTypes; + std::vector<NScheme::TTypeInfo> KeyColumnTypes; }; private: @@ -337,7 +337,7 @@ private: batch.clear(); batch.reserve(Results.size()); - std::vector<NKikimr::NScheme::TTypeId> columnTypes; + std::vector<NKikimr::NScheme::TTypeInfo> columnTypes; columnTypes.reserve(Columns.size()); for (const auto& column : Columns) { auto colIt = TableScheme->ColumnsByName.find(column); diff --git a/ydb/core/kqp/runtime/kqp_tasks_runner.cpp b/ydb/core/kqp/runtime/kqp_tasks_runner.cpp index 2d555fbc006..223feb970ac 100644 --- a/ydb/core/kqp/runtime/kqp_tasks_runner.cpp +++ b/ydb/core/kqp/runtime/kqp_tasks_runner.cpp @@ -26,15 +26,17 @@ IDqOutputConsumer::TPtr KqpBuildOutputConsumer(const NDqProto::TTaskOutput& outp { switch (outputDesc.GetTypeCase()) { case NDqProto::TTaskOutput::kRangePartition: { - TVector<NUdf::TDataTypeId> keyColumnTypeIds; - keyColumnTypeIds.reserve(outputDesc.GetRangePartition().GetKeyColumns().size()); + TVector<NScheme::TTypeInfo> keyColumnTypeInfos; + keyColumnTypeInfos.reserve(outputDesc.GetRangePartition().GetKeyColumns().size()); TVector<TType*> keyColumnTypes; TVector<ui32> keyColumnIndices; GetColumnsInfo(type, outputDesc.GetRangePartition().GetKeyColumns(), keyColumnTypes, keyColumnIndices); YQL_ENSURE(!keyColumnTypes.empty()); - std::transform(keyColumnTypes.begin(), keyColumnTypes.end(), back_inserter(keyColumnTypeIds), [](const auto& tyPtr) { + std::transform(keyColumnTypes.begin(), keyColumnTypes.end(), back_inserter(keyColumnTypeInfos), [](const auto& tyPtr) { + // TODO: support pg types YQL_ENSURE(tyPtr->GetKind() == NKikimr::NMiniKQL::TType::EKind::Data); - return static_cast<NKikimr::NMiniKQL::TDataType&>(*tyPtr).GetSchemeType(); + auto dataTypeId = static_cast<NKikimr::NMiniKQL::TDataType&>(*tyPtr).GetSchemeType(); + return NScheme::TTypeInfo((NScheme::TTypeId)dataTypeId); }); TVector<TKqpRangePartition> partitions; @@ -51,7 +53,7 @@ IDqOutputConsumer::TPtr KqpBuildOutputConsumer(const NDqProto::TTaskOutput& outp } return CreateOutputRangePartitionConsumer(std::move(outputs), std::move(partitions), - std::move(keyColumnTypeIds), std::move(keyColumnIndices), typeEnv); + std::move(keyColumnTypeInfos), std::move(keyColumnIndices), typeEnv); } case NDqProto::TTaskOutput::kEffects: { diff --git a/ydb/core/kqp/runtime/ut/CMakeLists.darwin.txt b/ydb/core/kqp/runtime/ut/CMakeLists.darwin.txt index 877a653f444..deae8ecaa2e 100644 --- a/ydb/core/kqp/runtime/ut/CMakeLists.darwin.txt +++ b/ydb/core/kqp/runtime/ut/CMakeLists.darwin.txt @@ -21,7 +21,7 @@ target_link_libraries(ydb-core-kqp-runtime-ut PUBLIC cpp-testing-unittest_main core-kqp-runtime cpp-testing-unittest - core-testlib-basics + testlib-basics-default udf-service-exception_policy ) target_link_options(ydb-core-kqp-runtime-ut PRIVATE diff --git a/ydb/core/kqp/runtime/ut/CMakeLists.linux.txt b/ydb/core/kqp/runtime/ut/CMakeLists.linux.txt index 8e5babbce0b..4c92ff9d5b3 100644 --- a/ydb/core/kqp/runtime/ut/CMakeLists.linux.txt +++ b/ydb/core/kqp/runtime/ut/CMakeLists.linux.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-kqp-runtime-ut PUBLIC cpp-testing-unittest_main core-kqp-runtime cpp-testing-unittest - core-testlib-basics + testlib-basics-default udf-service-exception_policy ) target_link_options(ydb-core-kqp-runtime-ut PRIVATE diff --git a/ydb/core/kqp/ut/CMakeLists.darwin.txt b/ydb/core/kqp/ut/CMakeLists.darwin.txt index f5fffdab6f6..7f2431236f4 100644 --- a/ydb/core/kqp/ut/CMakeLists.darwin.txt +++ b/ydb/core/kqp/ut/CMakeLists.darwin.txt @@ -8,6 +8,7 @@ add_subdirectory(common) add_subdirectory(fat) +add_subdirectory(pg) add_subdirectory(spilling) add_executable(ydb-core-kqp-ut) @@ -31,6 +32,7 @@ target_link_libraries(ydb-core-kqp-ut PUBLIC kqp-ut-common cpp-client-draft cpp-client-ydb_proto + yql-sql-pg_dummy re2_udf ) target_link_options(ydb-core-kqp-ut PRIVATE diff --git a/ydb/core/kqp/ut/CMakeLists.linux.txt b/ydb/core/kqp/ut/CMakeLists.linux.txt index af6b41955cb..c5fa765a09b 100644 --- a/ydb/core/kqp/ut/CMakeLists.linux.txt +++ b/ydb/core/kqp/ut/CMakeLists.linux.txt @@ -8,6 +8,7 @@ add_subdirectory(common) add_subdirectory(fat) +add_subdirectory(pg) add_subdirectory(spilling) add_executable(ydb-core-kqp-ut) @@ -33,6 +34,7 @@ target_link_libraries(ydb-core-kqp-ut PUBLIC kqp-ut-common cpp-client-draft cpp-client-ydb_proto + yql-sql-pg_dummy re2_udf ) target_link_options(ydb-core-kqp-ut PRIVATE diff --git a/ydb/core/kqp/ut/fat/CMakeLists.darwin.txt b/ydb/core/kqp/ut/fat/CMakeLists.darwin.txt index 975cf7d001a..bfc9ff61e44 100644 --- a/ydb/core/kqp/ut/fat/CMakeLists.darwin.txt +++ b/ydb/core/kqp/ut/fat/CMakeLists.darwin.txt @@ -25,6 +25,7 @@ target_link_libraries(ydb-core-kqp-ut-fat PUBLIC core-kqp-host core-kqp-provider kqp-ut-common + yql-sql-pg_dummy ) target_link_options(ydb-core-kqp-ut-fat PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/kqp/ut/fat/CMakeLists.linux.txt b/ydb/core/kqp/ut/fat/CMakeLists.linux.txt index f372e3bf900..48c75b49f72 100644 --- a/ydb/core/kqp/ut/fat/CMakeLists.linux.txt +++ b/ydb/core/kqp/ut/fat/CMakeLists.linux.txt @@ -27,6 +27,7 @@ target_link_libraries(ydb-core-kqp-ut-fat PUBLIC core-kqp-host core-kqp-provider kqp-ut-common + yql-sql-pg_dummy ) target_link_options(ydb-core-kqp-ut-fat PRIVATE -ldl diff --git a/ydb/core/kqp/ut/pg/CMakeLists.darwin.txt b/ydb/core/kqp/ut/pg/CMakeLists.darwin.txt new file mode 100644 index 00000000000..d9be4ac2454 --- /dev/null +++ b/ydb/core/kqp/ut/pg/CMakeLists.darwin.txt @@ -0,0 +1,51 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-core-kqp-ut-pg) +target_compile_options(ydb-core-kqp-ut-pg PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION + -Wno-everything +) +target_include_directories(ydb-core-kqp-ut-pg PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/kqp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/include +) +target_link_libraries(ydb-core-kqp-ut-pg PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + ydb-core-kqp + kqp-ut-common + udf-service-exception_policy + yql-sql-pg +) +target_link_options(ydb-core-kqp-ut-pg PRIVATE + -Wl,-no_deduplicate + -Wl,-sdk_version,10.15 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-core-kqp-ut-pg PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp +) +add_test( + NAME + ydb-core-kqp-ut-pg + COMMAND + ydb-core-kqp-ut-pg + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +vcs_info(ydb-core-kqp-ut-pg) diff --git a/ydb/core/kqp/ut/pg/CMakeLists.linux.txt b/ydb/core/kqp/ut/pg/CMakeLists.linux.txt new file mode 100644 index 00000000000..7508c3525ff --- /dev/null +++ b/ydb/core/kqp/ut/pg/CMakeLists.linux.txt @@ -0,0 +1,55 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-core-kqp-ut-pg) +target_compile_options(ydb-core-kqp-ut-pg PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION + -Wno-everything +) +target_include_directories(ydb-core-kqp-ut-pg PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/kqp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/include +) +target_link_libraries(ydb-core-kqp-ut-pg PUBLIC + contrib-libs-cxxsupp + yutil + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache + library-cpp-cpuid_check + cpp-testing-unittest_main + ydb-core-kqp + kqp-ut-common + udf-service-exception_policy + yql-sql-pg +) +target_link_options(ydb-core-kqp-ut-pg PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-core-kqp-ut-pg PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp +) +add_test( + NAME + ydb-core-kqp-ut-pg + COMMAND + ydb-core-kqp-ut-pg + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +vcs_info(ydb-core-kqp-ut-pg) diff --git a/ydb/core/kqp/ut/pg/CMakeLists.txt b/ydb/core/kqp/ut/pg/CMakeLists.txt new file mode 100644 index 00000000000..79468a5d8d0 --- /dev/null +++ b/ydb/core/kqp/ut/pg/CMakeLists.txt @@ -0,0 +1,13 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (APPLE) + include(CMakeLists.darwin.txt) +elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND UNIX AND NOT APPLE AND NOT ANDROID) + include(CMakeLists.linux.txt) +endif() diff --git a/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp b/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp new file mode 100644 index 00000000000..a8cb5602b9b --- /dev/null +++ b/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp @@ -0,0 +1,162 @@ +#include <ydb/core/kqp/ut/common/kqp_ut_common.h> + +extern "C" { +#include "postgres.h" +#include "catalog/pg_type_d.h" +} + +namespace NKikimr { +namespace NKqp { + +using namespace NYdb; +using namespace NYdb::NTable; + +Y_UNIT_TEST_SUITE(KqpPg) { + + auto makePgType = [] (ui32 oid, i32 typlen = -1) { return TPgType(oid, typlen, -1); }; + + Y_UNIT_TEST_NEW_ENGINE(CreateTableBulkUpsertAndRead) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + auto builder = TTableBuilder() + .AddNullableColumn("boolean", makePgType(BOOLOID)) + .AddNullableColumn("char", makePgType(CHAROID)) + .AddNullableColumn("int2", makePgType(INT2OID)) + .AddNullableColumn("int4", makePgType(INT4OID)) + .AddNullableColumn("int8", makePgType(INT8OID)) + .AddNullableColumn("float4", makePgType(FLOAT4OID)) + .AddNullableColumn("float8", makePgType(FLOAT8OID)) + .AddNullableColumn("text", makePgType(TEXTOID)) + .AddNullableColumn("bytea", makePgType(BYTEAOID)) + .AddNullableColumn("bpchar", makePgType(BPCHAROID)) + .AddNullableColumn("varchar", makePgType(VARCHAROID)) + .SetPrimaryKeyColumn("int2") + .SetPrimaryKeyColumn("int4") + .SetPrimaryKeyColumn("int8") + .SetPrimaryKeyColumn("float4") + .SetPrimaryKeyColumn("float8") + .SetPrimaryKeyColumn("text") + .SetPrimaryKeyColumn("bytea") + .SetPrimaryKeyColumn("bpchar"); + + auto result = session.CreateTable("/Root/Pg", builder.Build()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + NYdb::TValueBuilder rows; + rows.BeginList(); + for (size_t i = 0; i < 10; ++i) { + auto boolVal = true; + TString boolStr((const char*)&boolVal, sizeof(boolVal)); + auto charVal = (char)i; + TString charStr((const char*)&charVal, sizeof(charVal)); + auto int2Val = (i16)i; + TString int2Str((const char*)&int2Val, sizeof(int2Val)); + auto int4Val = (i32)i; + TString int4Str((const char*)&int4Val, sizeof(int4Val)); + auto int8Val = (i64)i; + TString int8Str((const char*)&int8Val, sizeof(int8Val)); + auto float4Val = (float)i; + TString float4Str((const char*)&float4Val, sizeof(float4Val)); + auto float8Val = (double)i; + TString float8Str((const char*)&float8Val, sizeof(float8Val)); + TString textStr = Sprintf("text %" PRIu64, i); + TString byteaStr = Sprintf("bytea %" PRIu64, i); + TString bpcharStr = Sprintf("bpchar %" PRIu64, i); + TString varcharStr = Sprintf("varchar %" PRIu64, i); + + rows.AddListItem() + .BeginStruct() + .AddMember("boolean").Pg(TPgValue(TPgValue::VK_BINARY, boolStr, makePgType(BOOLOID))) + .AddMember("char").Pg(TPgValue(TPgValue::VK_BINARY, charStr, makePgType(CHAROID))) + .AddMember("int2").Pg(TPgValue(TPgValue::VK_BINARY, int2Str, makePgType(INT2OID))) + .AddMember("int4").Pg(TPgValue(TPgValue::VK_BINARY, int4Str, makePgType(INT4OID))) + .AddMember("int8").Pg(TPgValue(TPgValue::VK_BINARY, int8Str, makePgType(INT8OID))) + .AddMember("float4").Pg(TPgValue(TPgValue::VK_BINARY, float4Str, makePgType(FLOAT4OID))) + .AddMember("float8").Pg(TPgValue(TPgValue::VK_BINARY, float8Str, makePgType(FLOAT8OID))) + .AddMember("text").Pg(TPgValue(TPgValue::VK_BINARY, textStr, makePgType(TEXTOID))) + .AddMember("bytea").Pg(TPgValue(TPgValue::VK_BINARY, byteaStr, makePgType(BYTEAOID))) + .AddMember("bpchar").Pg(TPgValue(TPgValue::VK_BINARY, bpcharStr, makePgType(BPCHAROID))) + .AddMember("varchar").Pg(TPgValue(TPgValue::VK_BINARY, varcharStr, makePgType(VARCHAROID))) + .EndStruct(); + } + rows.EndList(); + + result = db.BulkUpsert("/Root/Pg", rows.Build()).GetValueSync();; + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + session.Close().GetValueSync(); + + auto readSettings = TReadTableSettings() + .AppendColumns("boolean") + .AppendColumns("char") + .AppendColumns("int2") + .AppendColumns("int4") + .AppendColumns("int8") + .AppendColumns("float4") + .AppendColumns("float8") + .AppendColumns("text") + .AppendColumns("bytea") + .AppendColumns("bpchar") + .AppendColumns("varchar"); + + auto it = session.ReadTable("/Root/Pg", readSettings).GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), result.GetIssues().ToString()); + + bool eos = false; + while (!eos) { + auto part = it.ReadNext().ExtractValueSync(); + if (!part.IsSuccess()) { + eos = true; + UNIT_ASSERT_C(part.EOS(), result.GetIssues().ToString()); + continue; + } + auto resultSet = part.ExtractPart(); + TResultSetParser parser(resultSet); + for (size_t i = 0; parser.TryNextRow(); ++i) { + auto boolVal = true; + TString boolStr((const char*)&boolVal, sizeof(boolVal)); + auto charVal = (char)i; + TString charStr((const char*)&charVal, sizeof(charVal)); + auto int2Val = (i16)i; + TString int2Str((const char*)&int2Val, sizeof(int2Val)); + auto int4Val = (i32)i; + TString int4Str((const char*)&int4Val, sizeof(int4Val)); + auto int8Val = (i64)i; + TString int8Str((const char*)&int8Val, sizeof(int8Val)); + auto float4Val = (float)i; + TString float4Str((const char*)&float4Val, sizeof(float4Val)); + auto float8Val = (double)i; + TString float8Str((const char*)&float8Val, sizeof(float8Val)); + TString textStr = Sprintf("text %" PRIu64, i); + TString byteaStr = Sprintf("bytea %" PRIu64, i); + TString bpcharStr = Sprintf("bpchar %" PRIu64, i); + TString varcharStr = Sprintf("varchar %" PRIu64, i); + +#define COLUMN(type, valStr) \ + { \ + auto& column = parser.ColumnParser(type); \ + column.OpenOptional(); \ + UNIT_ASSERT_VALUES_EQUAL(valStr, column.GetPg().Content_); \ + column.CloseOptional(); \ + } + COLUMN("boolean", boolStr); + COLUMN("char", charStr); + COLUMN("int2", int2Str); + COLUMN("int4", int4Str); + COLUMN("int8", int8Str); + COLUMN("float4", float4Str); + COLUMN("float8", float8Str); + COLUMN("text", textStr); + COLUMN("bytea", byteaStr); + COLUMN("bpchar", bpcharStr); + COLUMN("varchar", varcharStr); +#undef COLUMN + } + } + } +} + +} // namespace NKqp +} // namespace NKikimr diff --git a/ydb/core/kqp/ut/spilling/CMakeLists.darwin.txt b/ydb/core/kqp/ut/spilling/CMakeLists.darwin.txt index 5f49554dc5b..26e90b05233 100644 --- a/ydb/core/kqp/ut/spilling/CMakeLists.darwin.txt +++ b/ydb/core/kqp/ut/spilling/CMakeLists.darwin.txt @@ -25,6 +25,7 @@ target_link_libraries(ydb-core-kqp-ut-spilling PUBLIC core-kqp-host core-kqp-provider kqp-ut-common + yql-sql-pg_dummy ) target_link_options(ydb-core-kqp-ut-spilling PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/kqp/ut/spilling/CMakeLists.linux.txt b/ydb/core/kqp/ut/spilling/CMakeLists.linux.txt index 9821c1b0403..2ce59f032a6 100644 --- a/ydb/core/kqp/ut/spilling/CMakeLists.linux.txt +++ b/ydb/core/kqp/ut/spilling/CMakeLists.linux.txt @@ -27,6 +27,7 @@ target_link_libraries(ydb-core-kqp-ut-spilling PUBLIC core-kqp-host core-kqp-provider kqp-ut-common + yql-sql-pg_dummy ) target_link_options(ydb-core-kqp-ut-spilling PRIVATE -ldl diff --git a/ydb/core/mind/address_classification/ut/CMakeLists.darwin.txt b/ydb/core/mind/address_classification/ut/CMakeLists.darwin.txt index 40e5b3b649a..7ba877b4966 100644 --- a/ydb/core/mind/address_classification/ut/CMakeLists.darwin.txt +++ b/ydb/core/mind/address_classification/ut/CMakeLists.darwin.txt @@ -21,7 +21,7 @@ target_link_libraries(ydb-core-mind-address_classification-ut PUBLIC cpp-testing-unittest_main core-mind-address_classification cpp-actors-http - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-mind-address_classification-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/mind/address_classification/ut/CMakeLists.linux.txt b/ydb/core/mind/address_classification/ut/CMakeLists.linux.txt index e3481d90092..98ff5aec614 100644 --- a/ydb/core/mind/address_classification/ut/CMakeLists.linux.txt +++ b/ydb/core/mind/address_classification/ut/CMakeLists.linux.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-mind-address_classification-ut PUBLIC cpp-testing-unittest_main core-mind-address_classification cpp-actors-http - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-mind-address_classification-ut PRIVATE -ldl diff --git a/ydb/core/mind/bscontroller/ut_bscontroller/CMakeLists.darwin.txt b/ydb/core/mind/bscontroller/ut_bscontroller/CMakeLists.darwin.txt index 0b9fa6d82c9..3c604e326e6 100644 --- a/ydb/core/mind/bscontroller/ut_bscontroller/CMakeLists.darwin.txt +++ b/ydb/core/mind/bscontroller/ut_bscontroller/CMakeLists.darwin.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-mind-bscontroller-ut_bscontroller PUBLIC blobstorage-dsproxy-mock core-mind-bscontroller ydb-core-protos - ydb-core-testlib + core-testlib-default core-testlib-basics ) target_link_options(ydb-core-mind-bscontroller-ut_bscontroller PRIVATE diff --git a/ydb/core/mind/bscontroller/ut_bscontroller/CMakeLists.linux.txt b/ydb/core/mind/bscontroller/ut_bscontroller/CMakeLists.linux.txt index da1f7e96ede..8b6d2c3e10e 100644 --- a/ydb/core/mind/bscontroller/ut_bscontroller/CMakeLists.linux.txt +++ b/ydb/core/mind/bscontroller/ut_bscontroller/CMakeLists.linux.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-mind-bscontroller-ut_bscontroller PUBLIC blobstorage-dsproxy-mock core-mind-bscontroller ydb-core-protos - ydb-core-testlib + core-testlib-default core-testlib-basics ) target_link_options(ydb-core-mind-bscontroller-ut_bscontroller PRIVATE diff --git a/ydb/core/mind/hive/ut/CMakeLists.darwin.txt b/ydb/core/mind/hive/ut/CMakeLists.darwin.txt index 5811cd42acc..21e6e84e0fa 100644 --- a/ydb/core/mind/hive/ut/CMakeLists.darwin.txt +++ b/ydb/core/mind/hive/ut/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-mind-hive-ut PUBLIC library-cpp-svnversion ydb-core-base ydb-core-mind - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-mind-hive-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/mind/hive/ut/CMakeLists.linux.txt b/ydb/core/mind/hive/ut/CMakeLists.linux.txt index 6ee3abd0529..cdd5a1718c7 100644 --- a/ydb/core/mind/hive/ut/CMakeLists.linux.txt +++ b/ydb/core/mind/hive/ut/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-mind-hive-ut PUBLIC library-cpp-svnversion ydb-core-base ydb-core-mind - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-mind-hive-ut PRIVATE -ldl diff --git a/ydb/core/mind/ut/CMakeLists.darwin.txt b/ydb/core/mind/ut/CMakeLists.darwin.txt index b8a7b1070e1..2e24d15e6c9 100644 --- a/ydb/core/mind/ut/CMakeLists.darwin.txt +++ b/ydb/core/mind/ut/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-mind-ut PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-mind-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/mind/ut/CMakeLists.linux.txt b/ydb/core/mind/ut/CMakeLists.linux.txt index bfbef7e3c7c..41d2ada1098 100644 --- a/ydb/core/mind/ut/CMakeLists.linux.txt +++ b/ydb/core/mind/ut/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-mind-ut PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-mind-ut PRIVATE -ldl diff --git a/ydb/core/mind/ut_fat/CMakeLists.darwin.txt b/ydb/core/mind/ut_fat/CMakeLists.darwin.txt index c72c32ff065..6630f84c0a6 100644 --- a/ydb/core/mind/ut_fat/CMakeLists.darwin.txt +++ b/ydb/core/mind/ut_fat/CMakeLists.darwin.txt @@ -29,7 +29,7 @@ target_link_libraries(ydb-core-mind-ut_fat PUBLIC core-blobstorage-nodewarden core-blobstorage-pdisk blobstorage-vdisk-common - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-mind-ut_fat PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/mind/ut_fat/CMakeLists.linux.txt b/ydb/core/mind/ut_fat/CMakeLists.linux.txt index 01bd8cc6566..e650145cefb 100644 --- a/ydb/core/mind/ut_fat/CMakeLists.linux.txt +++ b/ydb/core/mind/ut_fat/CMakeLists.linux.txt @@ -31,7 +31,7 @@ target_link_libraries(ydb-core-mind-ut_fat PUBLIC core-blobstorage-nodewarden core-blobstorage-pdisk blobstorage-vdisk-common - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-mind-ut_fat PRIVATE -ldl diff --git a/ydb/core/persqueue/pq_impl.cpp b/ydb/core/persqueue/pq_impl.cpp index 9160db6f6c7..08654c1e7ef 100644 --- a/ydb/core/persqueue/pq_impl.cpp +++ b/ydb/core/persqueue/pq_impl.cpp @@ -8,6 +8,7 @@ #include <ydb/core/protos/pqconfig.pb.h> #include <ydb/core/protos/counters_keyvalue.pb.h> #include <ydb/core/metering/metering.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/sys_view/service/sysview_service.h> #include <ydb/core/tablet/tablet_counters.h> #include <library/cpp/json/json_writer.h> @@ -633,7 +634,9 @@ void TPersQueue::ApplyNewConfigAndReply(const TActorContext& ctx) KeySchema.clear(); KeySchema.reserve(Config.PartitionKeySchemaSize()); for (const auto& component : Config.GetPartitionKeySchema()) { - KeySchema.push_back(component.GetTypeId()); + auto typeInfo = NScheme::TypeInfoFromProtoColumnType(component.GetTypeId(), + component.HasTypeInfo() ? &component.GetTypeInfo() : nullptr); + KeySchema.push_back(typeInfo); } Y_VERIFY(TopicName.size(), "Need topic name here"); @@ -749,7 +752,9 @@ void TPersQueue::ReadConfig(const NKikimrClient::TKeyValueResponse::TReadResult& KeySchema.clear(); KeySchema.reserve(Config.PartitionKeySchemaSize()); for (const auto& component : Config.GetPartitionKeySchema()) { - KeySchema.push_back(component.GetTypeId()); + auto typeInfo = NScheme::TypeInfoFromProtoColumnType(component.GetTypeId(), + component.HasTypeInfo() ? &component.GetTypeInfo() : nullptr); + KeySchema.push_back(typeInfo); } ui32 cacheSize = CACHE_SIZE; diff --git a/ydb/core/persqueue/pq_impl.h b/ydb/core/persqueue/pq_impl.h index 73fa4f3cd0b..2b9c9cbe975 100644 --- a/ydb/core/persqueue/pq_impl.h +++ b/ydb/core/persqueue/pq_impl.h @@ -150,7 +150,7 @@ private: NPersQueue::TTopicConverterPtr TopicConverter; bool IsLocalDC; TString DCId; - TVector<NScheme::TTypeId> KeySchema; + TVector<NScheme::TTypeInfo> KeySchema; NKikimrPQ::TPQTabletConfig Config; NKikimrPQ::ETabletState TabletState; diff --git a/ydb/core/persqueue/ut/CMakeLists.darwin.txt b/ydb/core/persqueue/ut/CMakeLists.darwin.txt index e3ab43c227d..0bb0a158ba0 100644 --- a/ydb/core/persqueue/ut/CMakeLists.darwin.txt +++ b/ydb/core/persqueue/ut/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-persqueue-ut PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb_persqueue_core-ut-ut_utils library-cpp-resource ) diff --git a/ydb/core/persqueue/ut/CMakeLists.linux.txt b/ydb/core/persqueue/ut/CMakeLists.linux.txt index 3caed2d6a36..4bcfc755eac 100644 --- a/ydb/core/persqueue/ut/CMakeLists.linux.txt +++ b/ydb/core/persqueue/ut/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-persqueue-ut PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb_persqueue_core-ut-ut_utils library-cpp-resource ) diff --git a/ydb/core/persqueue/ut_slow/CMakeLists.darwin.txt b/ydb/core/persqueue/ut_slow/CMakeLists.darwin.txt index b4a546fecf3..a4ad0987487 100644 --- a/ydb/core/persqueue/ut_slow/CMakeLists.darwin.txt +++ b/ydb/core/persqueue/ut_slow/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-persqueue-ut_slow PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-persqueue-ut_slow PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/persqueue/ut_slow/CMakeLists.linux.txt b/ydb/core/persqueue/ut_slow/CMakeLists.linux.txt index 038edb3538a..302203bfa40 100644 --- a/ydb/core/persqueue/ut_slow/CMakeLists.linux.txt +++ b/ydb/core/persqueue/ut_slow/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-persqueue-ut_slow PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-persqueue-ut_slow PRIVATE -ldl diff --git a/ydb/core/protos/CMakeLists.txt b/ydb/core/protos/CMakeLists.txt index d8e4f567463..3be4b8e5c2d 100644 --- a/ydb/core/protos/CMakeLists.txt +++ b/ydb/core/protos/CMakeLists.txt @@ -138,6 +138,7 @@ target_proto_messages(ydb-core-protos PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/protos/tx_proxy.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/tx_scheme.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/tx_sequenceshard.proto + ${CMAKE_SOURCE_DIR}/ydb/core/protos/type_info.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/pdiskfit.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/pqconfig.proto ${CMAKE_SOURCE_DIR}/ydb/core/protos/auth.proto diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index fb78fceeb76..7f4d35655c5 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -1,4 +1,5 @@ import "ydb/core/protos/base.proto"; +import "ydb/core/protos/type_info.proto"; import "ydb/core/protos/bind_channel_storage_pool.proto"; import "ydb/core/protos/pqconfig.proto"; import "ydb/core/protos/replication.proto"; @@ -76,6 +77,7 @@ message TColumnDescription { optional string Name = 1; optional string Type = 2; optional uint32 TypeId = 3; + optional NKikimrProto.TTypeInfo TypeInfo = 9; optional uint32 Id = 4; optional uint32 Family = 5; // On default place column to default(0) family optional string FamilyName = 6; // set Family by name (0 - "default") @@ -372,6 +374,7 @@ message TOlapColumnDescription { optional string Type = 3; optional uint32 TypeId = 4; // TypeId cannot be set explicitly, use Type + optional NKikimrProto.TTypeInfo TypeInfo = 6; // TODO: Intermediate codecs such as Delta, DubleDelta, dictionary encoding, etc. //TCompressionOptions Compression = 5; diff --git a/ydb/core/protos/flat_tx_scheme.proto b/ydb/core/protos/flat_tx_scheme.proto index 42cef3df5b8..ebced3f665b 100644 --- a/ydb/core/protos/flat_tx_scheme.proto +++ b/ydb/core/protos/flat_tx_scheme.proto @@ -1,5 +1,6 @@ import "library/cpp/actors/protos/actors.proto"; import "ydb/core/protos/base.proto"; +import "ydb/core/protos/type_info.proto"; import "ydb/core/protos/subdomains.proto"; import "ydb/core/protos/bind_channel_storage_pool.proto"; import "ydb/core/protos/flat_scheme_op.proto"; @@ -211,6 +212,7 @@ message TMigrateColumn { optional uint32 Id = 1; optional string Name = 2; optional uint32 ColType = 3; + optional NKikimrProto.TTypeInfo ColTypeInfo = 11; optional uint32 ColKeyOrder = 4; optional uint64 CreateVersion = 5; optional uint64 DeleteVersion = 6; diff --git a/ydb/core/protos/kqp.proto b/ydb/core/protos/kqp.proto index 56e65aa696b..8476ed6d0a0 100644 --- a/ydb/core/protos/kqp.proto +++ b/ydb/core/protos/kqp.proto @@ -3,6 +3,7 @@ option java_package = "ru.yandex.kikimr.proto"; option cc_enable_arenas = true; import "library/cpp/actors/protos/actors.proto"; +import "ydb/core/protos/type_info.proto"; import "ydb/core/protos/kqp_physical.proto"; import "ydb/core/protos/kqp_stats.proto"; import "ydb/core/protos/query_stats.proto"; @@ -124,6 +125,7 @@ message TKqpColumnMetadataProto { optional uint32 Id = 2; optional string Type = 3; optional uint32 TypeId = 4; + optional NKikimrProto.TTypeInfo TypeInfo = 7; repeated string Family = 5; optional bool NotNull = 6 [default = false]; }; diff --git a/ydb/core/protos/pqconfig.proto b/ydb/core/protos/pqconfig.proto index acd2f426442..22f9d1217f1 100644 --- a/ydb/core/protos/pqconfig.proto +++ b/ydb/core/protos/pqconfig.proto @@ -6,6 +6,7 @@ import "ydb/core/protos/msgbus_kv.proto"; import "ydb/core/protos/node_limits.proto"; import "ydb/core/protos/netclassifier.proto"; import "ydb/core/protos/services.proto"; +import "ydb/core/protos/type_info.proto"; import "library/cpp/actors/protos/actors.proto"; @@ -299,6 +300,7 @@ message TPQTabletConfig { message TKeyComponentSchema { optional string Name = 1; optional uint32 TypeId = 2; + optional NKikimrProto.TTypeInfo TypeInfo = 3; } repeated TKeyComponentSchema PartitionKeySchema = 30; diff --git a/ydb/core/protos/scheme_log.proto b/ydb/core/protos/scheme_log.proto index fcf4ba52b94..1507ed3cd00 100644 --- a/ydb/core/protos/scheme_log.proto +++ b/ydb/core/protos/scheme_log.proto @@ -1,4 +1,5 @@ import "ydb/core/protos/flat_scheme_op.proto"; +import "ydb/core/protos/type_info.proto"; package NTabletFlatScheme; option java_package = "ru.yandex.kikimr.proto"; @@ -57,6 +58,7 @@ message TAlterRecord { optional string ColumnName = 5; optional uint32 ColumnType = 6; + optional NKikimrProto.TTypeInfo ColumnTypeInfo = 25; optional bytes Default = 10; // Serialized default value for cell optional bool NotNull = 24 [default = false]; diff --git a/ydb/core/protos/tx_datashard.proto b/ydb/core/protos/tx_datashard.proto index 6ad3b0942be..f002b678fa4 100644 --- a/ydb/core/protos/tx_datashard.proto +++ b/ydb/core/protos/tx_datashard.proto @@ -3,6 +3,7 @@ option cc_enable_arenas = true; import "library/cpp/actors/protos/actors.proto"; import "ydb/core/protos/base.proto"; +import "ydb/core/protos/type_info.proto"; import "ydb/core/protos/kqp.proto"; import "ydb/core/protos/ssa.proto"; import "ydb/core/protos/tablet.proto"; @@ -102,6 +103,7 @@ message TReadTableTransaction { optional uint32 Id = 1; optional string Name = 2; optional uint32 TypeId = 3; + optional NKikimrProto.TTypeInfo TypeInfo = 4; } optional TTableId TableId = 1; @@ -148,6 +150,7 @@ message TKqpTransaction { optional uint32 Id = 1; optional string Name = 2; optional uint32 Type = 3; + optional NKikimrProto.TTypeInfo TypeInfo = 4; } message TColumnWriteMeta { @@ -213,6 +216,7 @@ message TKqpTransaction { optional TTableMeta Table = 1; repeated TColumnMeta Columns = 2; repeated uint32 KeyColumnTypes = 3; // for debug logs only + repeated NKikimrProto.TTypeInfo KeyColumnTypeInfos = 13; // for debug logs only repeated bool SkipNullKeys = 4; repeated TReadOpMeta Reads = 5; optional uint64 ItemsLimit = 6; @@ -1412,6 +1416,7 @@ message TEvKqpScan { optional uint64 SchemaVersion = 5; repeated uint32 ColumnTags = 6; repeated uint32 ColumnTypes = 7; + repeated NKikimrProto.TTypeInfo ColumnTypeInfos = 22; repeated bool SkipNullKeys = 8; repeated NKikimrTx.TKeyRange Ranges = 9; optional NKikimrKqp.TKqpSnapshot Snapshot = 10; diff --git a/ydb/core/protos/type_info.proto b/ydb/core/protos/type_info.proto new file mode 100644 index 00000000000..669e0f5ee7b --- /dev/null +++ b/ydb/core/protos/type_info.proto @@ -0,0 +1,6 @@ +package NKikimrProto; +option java_package = "ru.yandex.kikimr.proto"; + +message TTypeInfo { + optional uint32 PgTypeId = 1; +} diff --git a/ydb/core/quoter/quoter_service_bandwidth_test/CMakeLists.darwin.txt b/ydb/core/quoter/quoter_service_bandwidth_test/CMakeLists.darwin.txt index 1af9c185540..9dc9c7b207c 100644 --- a/ydb/core/quoter/quoter_service_bandwidth_test/CMakeLists.darwin.txt +++ b/ydb/core/quoter/quoter_service_bandwidth_test/CMakeLists.darwin.txt @@ -20,7 +20,7 @@ target_link_libraries(quoter_service_bandwidth_test PUBLIC ydb-core-base core-kesus-tablet ydb-core-quoter - ydb-core-testlib + core-testlib-default ) target_link_options(quoter_service_bandwidth_test PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/quoter/quoter_service_bandwidth_test/CMakeLists.linux.txt b/ydb/core/quoter/quoter_service_bandwidth_test/CMakeLists.linux.txt index f374fe45901..d493f1877a0 100644 --- a/ydb/core/quoter/quoter_service_bandwidth_test/CMakeLists.linux.txt +++ b/ydb/core/quoter/quoter_service_bandwidth_test/CMakeLists.linux.txt @@ -22,7 +22,7 @@ target_link_libraries(quoter_service_bandwidth_test PUBLIC ydb-core-base core-kesus-tablet ydb-core-quoter - ydb-core-testlib + core-testlib-default ) target_link_options(quoter_service_bandwidth_test PRIVATE -ldl diff --git a/ydb/core/quoter/ut/CMakeLists.darwin.txt b/ydb/core/quoter/ut/CMakeLists.darwin.txt index 07877e85689..034a8724441 100644 --- a/ydb/core/quoter/ut/CMakeLists.darwin.txt +++ b/ydb/core/quoter/ut/CMakeLists.darwin.txt @@ -21,7 +21,7 @@ target_link_libraries(ydb-core-quoter-ut PUBLIC cpp-testing-unittest_main ydb-core-quoter cpp-testing-gmock_in_unittest - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-quoter-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/quoter/ut/CMakeLists.linux.txt b/ydb/core/quoter/ut/CMakeLists.linux.txt index b9b4aa78c8d..80ac2b075da 100644 --- a/ydb/core/quoter/ut/CMakeLists.linux.txt +++ b/ydb/core/quoter/ut/CMakeLists.linux.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-quoter-ut PUBLIC cpp-testing-unittest_main ydb-core-quoter cpp-testing-gmock_in_unittest - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-quoter-ut PRIVATE -ldl diff --git a/ydb/core/scheme/CMakeLists.txt b/ydb/core/scheme/CMakeLists.txt index 78d15109ff4..1b70a91472d 100644 --- a/ydb/core/scheme/CMakeLists.txt +++ b/ydb/core/scheme/CMakeLists.txt @@ -28,4 +28,6 @@ target_sources(ydb-core-scheme PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/scheme/scheme_tablecell.cpp ${CMAKE_SOURCE_DIR}/ydb/core/scheme/scheme_tabledefs.cpp ${CMAKE_SOURCE_DIR}/ydb/core/scheme/scheme_types_defs.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/scheme/scheme_type_info.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/scheme/scheme_types_proto.cpp ) diff --git a/ydb/core/scheme/scheme_borders.cpp b/ydb/core/scheme/scheme_borders.cpp index f3089b53ace..b4407fafe07 100644 --- a/ydb/core/scheme/scheme_borders.cpp +++ b/ydb/core/scheme/scheme_borders.cpp @@ -17,7 +17,7 @@ namespace NKikimr { } int ComparePrefixBorders( - TConstArrayRef<NScheme::TTypeId> keyTypes, + TConstArrayRef<NScheme::TTypeInfo> keyTypes, TConstArrayRef<TCell> left, EPrefixMode leftMode, TConstArrayRef<TCell> right, EPrefixMode rightMode) { diff --git a/ydb/core/scheme/scheme_borders.h b/ydb/core/scheme/scheme_borders.h index 709af56aca8..ab9b3fa8a64 100644 --- a/ydb/core/scheme/scheme_borders.h +++ b/ydb/core/scheme/scheme_borders.h @@ -23,7 +23,7 @@ namespace NKikimr { * Compares two prefix borders */ int ComparePrefixBorders( - TConstArrayRef<NScheme::TTypeId> keyTypes, + TConstArrayRef<NScheme::TTypeInfo> keyTypes, TConstArrayRef<TCell> left, EPrefixMode leftMode, TConstArrayRef<TCell> right, EPrefixMode rightMode); diff --git a/ydb/core/scheme/scheme_borders_ut.cpp b/ydb/core/scheme/scheme_borders_ut.cpp index 59fefd3b0a0..e27367fb4ec 100644 --- a/ydb/core/scheme/scheme_borders_ut.cpp +++ b/ydb/core/scheme/scheme_borders_ut.cpp @@ -13,9 +13,9 @@ Y_UNIT_TEST_SUITE(SchemeBorders) { class THelper { public: explicit THelper(size_t keyCount) { - Types.resize(keyCount); + Types.reserve(keyCount); for (size_t i = 0; i < keyCount; ++i) { - Types[i] = NScheme::NTypeIds::Uint32; + Types.push_back(NScheme::TTypeInfo(NScheme::NTypeIds::Uint32)); } } @@ -127,7 +127,7 @@ Y_UNIT_TEST_SUITE(SchemeBorders) { } private: - TVector<NScheme::TTypeId> Types; + TVector<NScheme::TTypeInfo> Types; TVector<TCell> LeftKey; TVector<TCell> RightKey; }; diff --git a/ydb/core/scheme/scheme_tablecell.cpp b/ydb/core/scheme/scheme_tablecell.cpp index c9ffaccc8e9..bd1069701a7 100644 --- a/ydb/core/scheme/scheme_tablecell.cpp +++ b/ydb/core/scheme/scheme_tablecell.cpp @@ -60,25 +60,32 @@ TOwnedCellVec::TInit TOwnedCellVec::Allocate(TOwnedCellVec::TCellVec cells) { }; } -TString DbgPrintCell(const TCell& r, NScheme::TTypeId typeId, const NScheme::TTypeRegistry ®) { - NScheme::ITypeSP t = reg.GetType(typeId); +TString DbgPrintCell(const TCell& r, NScheme::TTypeInfo typeInfo, const NScheme::TTypeRegistry ®) { + auto typeId = typeInfo.GetTypeId(); + TString res; - if (!t.IsKnownType()) - return Sprintf("Unknow typeId 0x%x", (ui32)typeId); + if (typeId == NScheme::NTypeIds::Pg) { + res = NPg::PgTypeNameFromTypeDesc(typeInfo.GetTypeDesc()); + } else { + NScheme::ITypeSP t = reg.GetType(typeId); - TString res = t->GetName(); - res += " : "; + if (!t.IsKnownType()) + return Sprintf("Unknow typeId 0x%x", (ui32)typeId); + + res = t->GetName(); + } - DbgPrintValue(res, r, typeId); + res += " : "; + DbgPrintValue(res, r, typeInfo); return res; } -void DbgPrintValue(TString &res, const TCell &r, ui32 type) { +void DbgPrintValue(TString &res, const TCell &r, NScheme::TTypeInfo typeInfo) { if (r.IsNull()) { res += "NULL"; } else { - switch (type) { + switch (typeInfo.GetTypeId()) { case NScheme::NTypeIds::Bool: res += r.AsValue<bool>() ? "true" : "false"; break; @@ -106,6 +113,9 @@ void DbgPrintValue(TString &res, const TCell &r, ui32 type) { case NScheme::NTypeIds::ActorId: res += ToString(r.AsValue<TActorId>()); break; + case NScheme::NTypeIds::Pg: + // TODO: support pg types + break; default: res += EscapeC(r.Data(), r.Size()); } diff --git a/ydb/core/scheme/scheme_tablecell.h b/ydb/core/scheme/scheme_tablecell.h index bccfdca97db..fffd827cf77 100644 --- a/ydb/core/scheme/scheme_tablecell.h +++ b/ydb/core/scheme/scheme_tablecell.h @@ -5,6 +5,8 @@ #include "scheme_type_order.h" #include "scheme_types_defs.h" +#include <ydb/library/yql/utils/pg_types.h> + #include <util/generic/hash.h> #include <util/system/unaligned_mem.h> @@ -119,7 +121,7 @@ using TCellsRef = TConstArrayRef<const TCell>; // NULL is considered equal to another NULL and less than non-NULL // ATTENTION!!! return value is int!! (NOT just -1,0,1) -inline int CompareTypedCells(const TCell& a, const TCell& b, NScheme::TTypeIdOrder type) { +inline int CompareTypedCells(const TCell& a, const TCell& b, NScheme::TTypeInfoOrder type) { using TPair = std::pair<ui64, ui64>; if (a.IsNull()) return b.IsNull() ? 0 : -1; @@ -188,6 +190,14 @@ inline int CompareTypedCells(const TCell& a, const TCell& b, NScheme::TTypeIdOrd return (va.second < vb.second) != type.IsDescending() ? -1 : 1; } + case NKikimr::NScheme::NTypeIds::Pg: + { + auto typeDesc = type.GetTypeDesc(); + Y_VERIFY(typeDesc, "no pg type descriptor"); + int result = NPg::PgNativeBinaryCompare(a.Data(), a.Size(), b.Data(), b.Size(), typeDesc); + return type.IsDescending() ? -result : result; + } + default: Y_VERIFY_DEBUG(false, "Unknown type"); }; @@ -225,11 +235,12 @@ inline int CompareTypedCellVectors(const TCell* a, const TCell* b, const TTypeCl } // TODO: use NYql ops when TCell and TUnboxedValuePod had merged -inline ui64 GetValueHash(NScheme::TTypeId type, const TCell& cell) { +inline ui64 GetValueHash(NScheme::TTypeInfo info, const TCell& cell) { if (cell.IsNull()) return 0; - const NYql::NProto::TypeIds yqlType = static_cast<NYql::NProto::TypeIds>(type); + auto typeId = info.GetTypeId(); + const NYql::NProto::TypeIds yqlType = static_cast<NYql::NProto::TypeIds>(typeId); switch (yqlType) { case NYql::NProto::TypeIds::Bool: return ((*(const ui8 *)cell.Data()) == 0) ? THash<ui8>()((ui8)0) : THash<ui8>()((ui8)1); @@ -273,17 +284,23 @@ inline ui64 GetValueHash(NScheme::TTypeId type, const TCell& cell) { return ComputeHash(TStringBuf{cell.Data(), cell.Size()}); default: - Y_VERIFY_DEBUG(false, "Type not supported for user columns: %d", type); break; } + if (typeId == NKikimr::NScheme::NTypeIds::Pg) { + auto typeDesc = info.GetTypeDesc(); + Y_VERIFY(typeDesc, "no pg type descriptor"); + return NPg::PgNativeBinaryHash(cell.Data(), cell.Size(), typeDesc); + } + + Y_VERIFY_DEBUG(false, "Type not supported for user columns: %d", typeId); return 0; } // Only references a vector of cells and corresponding types // Doesn't own the memory struct TDbTupleRef { - const NKikimr::NScheme::TTypeId* Types; + const NKikimr::NScheme::TTypeInfo* Types; const TCell* Columns; ui32 ColumnCount; @@ -291,7 +308,7 @@ struct TDbTupleRef { return { Columns, ColumnCount }; } - TDbTupleRef(const NScheme::TTypeId* types = nullptr, const TCell* storage = nullptr, ui32 colCnt = 0) + TDbTupleRef(const NScheme::TTypeInfo* types = nullptr, const TCell* storage = nullptr, ui32 colCnt = 0) : Types(types) , Columns(storage) , ColumnCount(colCnt) @@ -523,8 +540,8 @@ private: TVector<TCell> Cells; }; -void DbgPrintValue(TString&, const TCell&, ui32 type); -TString DbgPrintCell(const TCell& r, NScheme::TTypeId typeId, const NScheme::TTypeRegistry& typeRegistry); +void DbgPrintValue(TString&, const TCell&, NScheme::TTypeInfo typeInfo); +TString DbgPrintCell(const TCell& r, NScheme::TTypeInfo typeInfo, const NScheme::TTypeRegistry& typeRegistry); TString DbgPrintTuple(const TDbTupleRef& row, const NScheme::TTypeRegistry& typeRegistry); } diff --git a/ydb/core/scheme/scheme_tablecell_ut.cpp b/ydb/core/scheme/scheme_tablecell_ut.cpp index 5bd4a27ab95..0b01da964b3 100644 --- a/ydb/core/scheme/scheme_tablecell_ut.cpp +++ b/ydb/core/scheme/scheme_tablecell_ut.cpp @@ -9,6 +9,9 @@ Y_UNIT_TEST_SUITE(Scheme) { using namespace NKikimr; + namespace NTypeIds = NScheme::NTypeIds; + using TTypeInfo = NScheme::TTypeInfo; + Y_UNIT_TEST(EmptyOwnedCellVec) { TOwnedCellVec empty; UNIT_ASSERT_VALUES_EQUAL(empty.size(), 0u); @@ -68,32 +71,32 @@ Y_UNIT_TEST_SUITE(Scheme) { double doubleVal = -0.0025; TVector<TCell> cells; - TVector<NScheme::TTypeId> types; + TVector<TTypeInfo> types; cells.push_back(TCell((const char*)&doubleVal, sizeof(doubleVal))); - types.push_back(NScheme::NTypeIds::Double); + types.push_back(TTypeInfo(NTypeIds::Double)); cells.push_back(TCell(smallStrVal, sizeof(smallStrVal))); - types.push_back(NScheme::NTypeIds::String); + types.push_back(TTypeInfo(NTypeIds::String)); cells.push_back(TCell()); - types.push_back(NScheme::NTypeIds::Utf8); + types.push_back(TTypeInfo(NTypeIds::Utf8)); cells.push_back(TCell(smallStrVal, sizeof(smallStrVal))); - types.push_back(NScheme::NTypeIds::Utf8); + types.push_back(TTypeInfo(NTypeIds::Utf8)); cells.push_back(TCell((const char*)&floatVal, sizeof(floatVal))); - types.push_back(NScheme::NTypeIds::Float); + types.push_back(TTypeInfo(NTypeIds::Float)); cells.push_back(TCell()); - types.push_back(NScheme::NTypeIds::Decimal); + types.push_back(TTypeInfo(NTypeIds::Decimal)); cells.push_back(TCell((const char*)&intVal, sizeof(ui64))); - types.push_back(NScheme::NTypeIds::Uint64); + types.push_back(TTypeInfo(NTypeIds::Uint64)); cells.push_back(TCell()); - types.push_back(NScheme::NTypeIds::Decimal); + types.push_back(TTypeInfo(NTypeIds::Decimal)); cells.push_back(TCell()); - types.push_back(NScheme::NTypeIds::Uint8); + types.push_back(TTypeInfo(NTypeIds::Uint8)); cells.push_back(TCell(bigStrVal, sizeof(bigStrVal))); - types.push_back(NScheme::NTypeIds::Utf8); + types.push_back(TTypeInfo(NTypeIds::Utf8)); cells.push_back(TCell()); - types.push_back(NScheme::NTypeIds::Double); + types.push_back(TTypeInfo(NTypeIds::Double)); cells.push_back(TCell((const char*)&intVal, sizeof(i32))); - types.push_back(NScheme::NTypeIds::Int32); + types.push_back(TTypeInfo(NTypeIds::Int32)); TSerializedCellVec vec(TSerializedCellVec::Serialize(cells)); @@ -210,10 +213,13 @@ Y_UNIT_TEST_SUITE(Scheme) { /** * CompareOrder test for cell1 < cell2 < cell3 given a type id */ - void DoTestCompareOrder(const TCell& cell1, const TCell& cell2, const TCell& cell3, NScheme::TTypeId type) { + void DoTestCompareOrder(const TCell& cell1, const TCell& cell2, const TCell& cell3, NScheme::TTypeId typeId) { TCell nullCell; - NScheme::TTypeIdOrder typeDescending(type, NScheme::EOrder::Descending); + NScheme::TTypeIdOrder typeIdDescending(typeId, NScheme::EOrder::Descending); + + NScheme::TTypeInfo type(typeId); + NScheme::TTypeInfoOrder typeDescending(typeIdDescending); // NULL is always equal to itself, both ascending and descending UNIT_ASSERT_EQUAL(CompareTypedCells(nullCell, nullCell, type), 0); @@ -286,38 +292,39 @@ Y_UNIT_TEST_SUITE(Scheme) { TArrayRef<const NScheme::TTypeId> yqlIds(NScheme::NTypeIds::YqlIds); for (NScheme::TTypeId typeId : yqlIds) { + NScheme::TTypeInfo typeInfo(typeId); switch (typeId) { case NScheme::NTypeIds::Int32: - GetValueHash(typeId, TCell(charArr, sizeof(i32))); - CompareTypedCells(TCell(charArr, sizeof(i32)), TCell(charArr, sizeof(i32)), typeId); + GetValueHash(typeInfo, TCell(charArr, sizeof(i32))); + CompareTypedCells(TCell(charArr, sizeof(i32)), TCell(charArr, sizeof(i32)), typeInfo); break; case NScheme::NTypeIds::Uint32: - GetValueHash(typeId, TCell(charArr, sizeof(ui32))); - CompareTypedCells(TCell(charArr, sizeof(ui32)), TCell(charArr, sizeof(ui32)), typeId); + GetValueHash(typeInfo, TCell(charArr, sizeof(ui32))); + CompareTypedCells(TCell(charArr, sizeof(ui32)), TCell(charArr, sizeof(ui32)), typeInfo); break; case NScheme::NTypeIds::Int64: - GetValueHash(typeId, TCell(charArr, sizeof(i64))); - CompareTypedCells(TCell(charArr, sizeof(i64)), TCell(charArr, sizeof(i64)), typeId); + GetValueHash(typeInfo, TCell(charArr, sizeof(i64))); + CompareTypedCells(TCell(charArr, sizeof(i64)), TCell(charArr, sizeof(i64)), typeInfo); break; case NScheme::NTypeIds::Uint64: - GetValueHash(typeId, TCell(charArr, sizeof(ui64))); - CompareTypedCells(TCell(charArr, sizeof(ui64)), TCell(charArr, sizeof(ui64)), typeId); + GetValueHash(typeInfo, TCell(charArr, sizeof(ui64))); + CompareTypedCells(TCell(charArr, sizeof(ui64)), TCell(charArr, sizeof(ui64)), typeInfo); break; case NScheme::NTypeIds::Byte: - GetValueHash(typeId, TCell(charArr, sizeof(ui8))); - CompareTypedCells(TCell(charArr, sizeof(ui8)), TCell(charArr, sizeof(ui8)), typeId); + GetValueHash(typeInfo, TCell(charArr, sizeof(ui8))); + CompareTypedCells(TCell(charArr, sizeof(ui8)), TCell(charArr, sizeof(ui8)), typeInfo); break; case NScheme::NTypeIds::Bool: - GetValueHash(typeId, TCell(charArr, sizeof(ui8))); - CompareTypedCells(TCell(charArr, sizeof(ui8)), TCell(charArr, sizeof(ui8)), typeId); + GetValueHash(typeInfo, TCell(charArr, sizeof(ui8))); + CompareTypedCells(TCell(charArr, sizeof(ui8)), TCell(charArr, sizeof(ui8)), typeInfo); break; case NScheme::NTypeIds::Double: - GetValueHash(typeId, TCell(charArr, sizeof(double))); - CompareTypedCells(TCell(charArr, sizeof(double)), TCell(charArr, sizeof(double)), typeId); + GetValueHash(typeInfo, TCell(charArr, sizeof(double))); + CompareTypedCells(TCell(charArr, sizeof(double)), TCell(charArr, sizeof(double)), typeInfo); break; case NScheme::NTypeIds::Float: - GetValueHash(typeId, TCell(charArr, sizeof(float))); - CompareTypedCells(TCell(charArr, sizeof(float)), TCell(charArr, sizeof(float)), typeId); + GetValueHash(typeInfo, TCell(charArr, sizeof(float))); + CompareTypedCells(TCell(charArr, sizeof(float)), TCell(charArr, sizeof(float)), typeInfo); break; case NScheme::NTypeIds::String: case NScheme::NTypeIds::Utf8: @@ -325,28 +332,28 @@ Y_UNIT_TEST_SUITE(Scheme) { case NScheme::NTypeIds::Json: case NScheme::NTypeIds::JsonDocument: case NScheme::NTypeIds::DyNumber: - GetValueHash(typeId, TCell(charArr, 30)); - CompareTypedCells(TCell(charArr, 30), TCell(charArr, 30), typeId); + GetValueHash(typeInfo, TCell(charArr, 30)); + CompareTypedCells(TCell(charArr, 30), TCell(charArr, 30), typeInfo); break; case NScheme::NTypeIds::Decimal: - GetValueHash(typeId, TCell(charArr, sizeof(ui64) * 2)); - CompareTypedCells(TCell(charArr, sizeof(ui64) * 2), TCell(charArr, sizeof(ui64) * 2), typeId); + GetValueHash(typeInfo, TCell(charArr, sizeof(ui64) * 2)); + CompareTypedCells(TCell(charArr, sizeof(ui64) * 2), TCell(charArr, sizeof(ui64) * 2), typeInfo); break; case NScheme::NTypeIds::Date: - GetValueHash(typeId, TCell(charArr, sizeof(ui16))); - CompareTypedCells(TCell(charArr, sizeof(ui16)), TCell(charArr, sizeof(ui16)), typeId); + GetValueHash(typeInfo, TCell(charArr, sizeof(ui16))); + CompareTypedCells(TCell(charArr, sizeof(ui16)), TCell(charArr, sizeof(ui16)), typeInfo); break; case NScheme::NTypeIds::Datetime: - GetValueHash(typeId, TCell(charArr, sizeof(ui32))); - CompareTypedCells(TCell(charArr, sizeof(ui32)), TCell(charArr, sizeof(ui32)), typeId); + GetValueHash(typeInfo, TCell(charArr, sizeof(ui32))); + CompareTypedCells(TCell(charArr, sizeof(ui32)), TCell(charArr, sizeof(ui32)), typeInfo); break; case NScheme::NTypeIds::Timestamp: - GetValueHash(typeId, TCell(charArr, sizeof(ui64))); - CompareTypedCells(TCell(charArr, sizeof(ui64)), TCell(charArr, sizeof(ui64)), typeId); + GetValueHash(typeInfo, TCell(charArr, sizeof(ui64))); + CompareTypedCells(TCell(charArr, sizeof(ui64)), TCell(charArr, sizeof(ui64)), typeInfo); break; case NScheme::NTypeIds::Interval: - GetValueHash(typeId, TCell(charArr, sizeof(i64))); - CompareTypedCells(TCell(charArr, sizeof(i64)), TCell(charArr, sizeof(i64)), typeId); + GetValueHash(typeInfo, TCell(charArr, sizeof(i64))); + CompareTypedCells(TCell(charArr, sizeof(i64)), TCell(charArr, sizeof(i64)), typeInfo); break; default: UNIT_FAIL("undefined YQL type"); diff --git a/ydb/core/scheme/scheme_tabledefs.cpp b/ydb/core/scheme/scheme_tabledefs.cpp index 672957e1fcc..e1d77d8dfa8 100644 --- a/ydb/core/scheme/scheme_tabledefs.cpp +++ b/ydb/core/scheme/scheme_tabledefs.cpp @@ -2,11 +2,11 @@ namespace NKikimr { -bool TTableRange::IsEmptyRange(TConstArrayRef<const NScheme::TTypeId> cellTypeIds) const { +bool TTableRange::IsEmptyRange(TConstArrayRef<const NScheme::TTypeInfo> types) const { if (Point) return false; - const int compares = CompareBorders<true, false>(To, From, InclusiveTo, InclusiveFrom, cellTypeIds); + const int compares = CompareBorders<true, false>(To, From, InclusiveTo, InclusiveFrom, types); return (compares < 0); } @@ -74,9 +74,9 @@ const char* TTableRange::IsAmbiguousReason(size_t keyColumnsCount) const noexcep return nullptr; } -bool TSerializedTableRange::IsEmpty(TConstArrayRef<NScheme::TTypeId> type) const +bool TSerializedTableRange::IsEmpty(TConstArrayRef<NScheme::TTypeInfo> types) const { - auto cmp = CompareBorders<true, false>(To.GetCells(), From.GetCells(), ToInclusive, FromInclusive, type); + auto cmp = CompareBorders<true, false>(To.GetCells(), From.GetCells(), ToInclusive, FromInclusive, types); return (cmp < 0); } diff --git a/ydb/core/scheme/scheme_tabledefs.h b/ydb/core/scheme/scheme_tabledefs.h index d7206c301f1..85ca8f929a0 100644 --- a/ydb/core/scheme/scheme_tabledefs.h +++ b/ydb/core/scheme/scheme_tabledefs.h @@ -179,7 +179,7 @@ public: } } - bool IsEmptyRange(TConstArrayRef<const NScheme::TTypeId> cellTypeIds) const; + bool IsEmptyRange(TConstArrayRef<const NScheme::TTypeInfo> types) const; bool IsFullRange(ui32 columnsCount) const; bool IsAmbiguous(size_t keyColumnsCount) const noexcept { @@ -248,7 +248,7 @@ public: } } - bool IsEmpty(TConstArrayRef<NScheme::TTypeId> type) const; + bool IsEmpty(TConstArrayRef<NScheme::TTypeInfo> types) const; TTableRange ToTableRange() const { return TTableRange(From.GetCells(), FromInclusive, To.GetCells(), ToInclusive, Point); @@ -297,7 +297,7 @@ int ComparePointAndRange(const TConstArrayRef<TCell>& point, const TTableRange& // of compared borders (or in other words upper range borders are compared). template<bool FirstLeft, bool SecondLeft> int CompareBorders(TConstArrayRef<TCell> first, TConstArrayRef<TCell> second, bool inclusiveFirst, bool inclusiveSecond, - TConstArrayRef<NScheme::TTypeId> cellTypes) + TConstArrayRef<NScheme::TTypeInfo> cellTypes) { const ui32 firstSize = first.size(); const ui32 secondSize = second.size(); @@ -346,7 +346,7 @@ int CompareBorders(TConstArrayRef<TCell> first, TConstArrayRef<TCell> second, bo /// @note returns 0 on any overlap inline int CompareRanges(const TTableRange& rangeX, const TTableRange& rangeY, - const TConstArrayRef<NScheme::TTypeId> types) + const TConstArrayRef<NScheme::TTypeInfo> types) { Y_VERIFY(!rangeX.Point); Y_VERIFY(!rangeY.Point); @@ -368,8 +368,8 @@ inline int CompareRanges(const TTableRange& rangeX, const TTableRange& rangeY, inline bool CheckRangesOverlap( const TTableRange& rangeX, const TTableRange& rangeY, - const TConstArrayRef<NScheme::TTypeId> typesX, - const TConstArrayRef<NScheme::TTypeId> typesY) + const TConstArrayRef<NScheme::TTypeInfo> typesX, + const TConstArrayRef<NScheme::TTypeInfo> typesY) { if (rangeX.Point && rangeY.Point) { // Works like ComparePointKeys @@ -635,7 +635,7 @@ public: struct TColumnOp { ui32 Column; EColumnOperation Operation; - ui32 ExpectedType; + NScheme::TTypeInfo ExpectedType; ui32 InplaceUpdateMode; ui32 ImmediateUpdateSize; }; @@ -643,7 +643,7 @@ public: // one column info (out) struct TColumnInfo { ui32 Column; - ui32 Type; + NScheme::TTypeInfo Type; ui32 AllowInplaceMode; EStatus Status; }; @@ -679,7 +679,7 @@ public: const TOwnedTableRange Range; const TRangeLimits RangeLimits; const ERowOperation RowOperation; - const TVector<NScheme::TTypeId> KeyColumnTypes; // For SelectRange there can be not full key + const TVector<NScheme::TTypeInfo> KeyColumnTypes; // For SelectRange there can be not full key const TVector<TColumnOp> Columns; const bool Reverse; TReadTarget ReadTarget; // Set for Read row operation diff --git a/ydb/core/scheme/scheme_type_info.cpp b/ydb/core/scheme/scheme_type_info.cpp new file mode 100644 index 00000000000..4d560fdbad1 --- /dev/null +++ b/ydb/core/scheme/scheme_type_info.cpp @@ -0,0 +1,16 @@ +#include "scheme_type_info.h" + +#include <ydb/public/lib/scheme_types/scheme_type_id.h> +#include <ydb/library/yql/utils/pg_types.h> + +namespace NKikimr::NScheme { + +const char* TypeName(const TTypeInfo typeInfo) { + if (typeInfo.GetTypeId() == NScheme::NTypeIds::Pg) { + return NPg::PgTypeNameFromTypeDesc(typeInfo.GetTypeDesc()); + } else { + return TypeName(typeInfo.GetTypeId()); + } +} + +} // namespace NKikimr::NScheme diff --git a/ydb/core/scheme/scheme_type_info.h b/ydb/core/scheme/scheme_type_info.h new file mode 100644 index 00000000000..f36f98112a6 --- /dev/null +++ b/ydb/core/scheme/scheme_type_info.h @@ -0,0 +1,9 @@ +#pragma once + +#include <ydb/core/scheme_types/scheme_type_info.h> + +namespace NKikimr::NScheme { + +const char* TypeName(const TTypeInfo typeInfo); + +} // NKikimr::NScheme diff --git a/ydb/core/scheme/scheme_type_order.h b/ydb/core/scheme/scheme_type_order.h index 2c1df0066d7..488ed7226f5 100644 --- a/ydb/core/scheme/scheme_type_order.h +++ b/ydb/core/scheme/scheme_type_order.h @@ -1,6 +1,7 @@ #pragma once #include "scheme_type_id.h" +#include "scheme_type_info.h" namespace NKikimr { namespace NScheme { @@ -50,5 +51,48 @@ private: ui16 Descending_ : 1; }; +struct TTypeInfoOrder { + TTypeInfoOrder() + {} + + TTypeInfoOrder(TTypeIdOrder typeIdOrder, void* typeDesc = {}) + : TypeIdOrder(typeIdOrder) + , TypeDesc(typeDesc) + {} + + TTypeInfoOrder(TTypeInfo typeInfo, EOrder order = EOrder::Ascending) + : TypeIdOrder(typeInfo.GetTypeId(), order) + , TypeDesc(typeInfo.GetTypeDesc()) + {} + + TTypeId GetTypeId() const { + return TypeIdOrder.GetTypeId(); + } + + EOrder GetOrder() const { + return TypeIdOrder.GetOrder(); + } + + bool IsAscending() const { + return TypeIdOrder.IsAscending(); + } + + bool IsDescending() const { + return TypeIdOrder.IsDescending(); + } + + void* GetTypeDesc() const { + return TypeDesc; + } + + TTypeInfo ToTypeInfo() const { + return TTypeInfo(GetTypeId(), GetTypeDesc()); + } + +private: + TTypeIdOrder TypeIdOrder; + void* TypeDesc = {}; +}; + } } diff --git a/ydb/core/scheme/scheme_types_defs.cpp b/ydb/core/scheme/scheme_types_defs.cpp index 8507a7d7df8..9014db47f00 100644 --- a/ydb/core/scheme/scheme_types_defs.cpp +++ b/ydb/core/scheme/scheme_types_defs.cpp @@ -15,10 +15,10 @@ namespace NNames { ::TString result; if (value) { - const ui32 fixedSize = GetFixedSize(value.Type()); + const ui32 fixedSize = GetFixedSize(value.TypeInfo()); if (fixedSize > 0 && value.Size() != fixedSize) { result = ::TStringBuilder() - << "Value with declared type " << ui16(value.Type()) + << "Value with declared type " << NScheme::TypeName(value.TypeInfo()) << " has unexpected size " << value.Size() << " (expected " << fixedSize << ")"; } @@ -27,14 +27,14 @@ namespace NNames { return result; } - ::TString HasUnexpectedValueSize(const ::NKikimr::TCell& value, TTypeId typeId) { + ::TString HasUnexpectedValueSize(const ::NKikimr::TCell& value, TTypeInfo typeInfo) { ::TString result; if (value) { - const ui32 fixedSize = GetFixedSize(typeId); + const ui32 fixedSize = GetFixedSize(typeInfo); if (fixedSize > 0 && value.Size() != fixedSize) { result = ::TStringBuilder() - << "Cell with declared type " << ui16(typeId) + << "Cell with declared type " << NScheme::TypeName(typeInfo) << " has unexpected size " << value.Size() << " (expected " << fixedSize << ")"; } diff --git a/ydb/core/scheme/scheme_types_defs.h b/ydb/core/scheme/scheme_types_defs.h index a0fd0f0e019..61373b59f77 100644 --- a/ydb/core/scheme/scheme_types_defs.h +++ b/ydb/core/scheme/scheme_types_defs.h @@ -4,6 +4,7 @@ #include <library/cpp/actors/core/actorid.h> #include <ydb/core/scheme_types/scheme_types_defs.h> +#include <ydb/library/yql/utils/pg_types.h> #include <util/stream/output.h> @@ -53,11 +54,13 @@ class TStepOrderId : public IIntegerPair<ui64, ui64, NTypeIds::StepOrderId, NNam //////////////////////////////////////////////////////// -inline ui32 GetFixedSize(NKikimr::NScheme::TTypeId typeId) { - switch (typeId) { +inline ui32 GetFixedSize(TTypeInfo typeInfo) { + switch (typeInfo.GetTypeId()) { #define KIKIMR_TYPE_MACRO(typeEnum, typeType, ...) case NTypeIds::typeEnum: return typeType::GetFixedSize(); KIKIMR_FOREACH_TYPE(KIKIMR_TYPE_MACRO) #undef KIKIMR_TYPE_MACRO + case NTypeIds::Pg: + return NPg::TypeDescGetTypeLen(typeInfo.GetTypeDesc()); default: return 0; } @@ -75,7 +78,7 @@ inline ui32 GetFixedSize(NKikimr::NScheme::TTypeId typeId) { * * Returns empty string on success or an error description in case of failure */ -::TString HasUnexpectedValueSize(const ::NKikimr::TCell& value, TTypeId typeId); +::TString HasUnexpectedValueSize(const ::NKikimr::TCell& value, TTypeInfo typeInfo); } // namespace NScheme } // namespace NKikimr diff --git a/ydb/core/scheme/scheme_types_proto.cpp b/ydb/core/scheme/scheme_types_proto.cpp new file mode 100644 index 00000000000..02bfbda2551 --- /dev/null +++ b/ydb/core/scheme/scheme_types_proto.cpp @@ -0,0 +1,26 @@ +#include "scheme_types_proto.h" +#include "scheme_tablecell.h" + +namespace NKikimr::NScheme { + +TProtoColumnType ProtoColumnTypeFromTypeInfo(const TTypeInfo typeInfo) { + TProtoColumnType columnType; + columnType.TypeId = (ui32)typeInfo.GetTypeId(); + if (typeInfo.GetTypeId() == NTypeIds::Pg) { + Y_VERIFY(typeInfo.GetTypeDesc(), "no pg type descriptor"); + columnType.TypeInfo = NKikimrProto::TTypeInfo(); + columnType.TypeInfo->SetPgTypeId(NPg::PgTypeIdFromTypeDesc(typeInfo.GetTypeDesc())); + } + return columnType; +} + +TTypeInfo TypeInfoFromProtoColumnType(ui32 typeId, const NKikimrProto::TTypeInfo* typeInfo) { + auto type = (TTypeId)typeId; + if (type == NTypeIds::Pg) { + Y_VERIFY(typeInfo); + return TTypeInfo(type, NPg::TypeDescFromPgTypeId(typeInfo->GetPgTypeId())); + } + return TTypeInfo(type); +} + +} // namespace NKikimr::NScheme diff --git a/ydb/core/scheme/scheme_types_proto.h b/ydb/core/scheme/scheme_types_proto.h new file mode 100644 index 00000000000..43c8789579c --- /dev/null +++ b/ydb/core/scheme/scheme_types_proto.h @@ -0,0 +1,17 @@ +#pragma once + +#include <ydb/core/scheme_types/scheme_type_info.h> +#include <ydb/core/protos/type_info.pb.h> + +namespace NKikimr::NScheme { + +struct TProtoColumnType { + ui32 TypeId = 0; + std::optional<NKikimrProto::TTypeInfo> TypeInfo; +}; + +TProtoColumnType ProtoColumnTypeFromTypeInfo(const TTypeInfo typeInfo); + +TTypeInfo TypeInfoFromProtoColumnType(ui32 typeId, const NKikimrProto::TTypeInfo* typeInfo); + +} // namespace NKikimr::NScheme diff --git a/ydb/core/scheme/ut/CMakeLists.darwin.txt b/ydb/core/scheme/ut/CMakeLists.darwin.txt index b4acf0aa966..73d0828c73f 100644 --- a/ydb/core/scheme/ut/CMakeLists.darwin.txt +++ b/ydb/core/scheme/ut/CMakeLists.darwin.txt @@ -8,6 +8,9 @@ add_executable(ydb-core-scheme-ut) +target_compile_options(ydb-core-scheme-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) target_include_directories(ydb-core-scheme-ut PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/scheme ) @@ -17,6 +20,8 @@ target_link_libraries(ydb-core-scheme-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main ydb-core-scheme + udf-service-stub + yql-sql-pg_dummy ) target_link_options(ydb-core-scheme-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/scheme/ut/CMakeLists.linux.txt b/ydb/core/scheme/ut/CMakeLists.linux.txt index 010617282f5..6061fdebdb9 100644 --- a/ydb/core/scheme/ut/CMakeLists.linux.txt +++ b/ydb/core/scheme/ut/CMakeLists.linux.txt @@ -8,6 +8,9 @@ add_executable(ydb-core-scheme-ut) +target_compile_options(ydb-core-scheme-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) target_include_directories(ydb-core-scheme-ut PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/scheme ) @@ -19,6 +22,8 @@ target_link_libraries(ydb-core-scheme-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main ydb-core-scheme + udf-service-stub + yql-sql-pg_dummy ) target_link_options(ydb-core-scheme-ut PRIVATE -ldl diff --git a/ydb/core/scheme_types/scheme_raw_type_value.h b/ydb/core/scheme_types/scheme_raw_type_value.h index 1cb0946b6d7..4dea1ceda4f 100644 --- a/ydb/core/scheme_types/scheme_raw_type_value.h +++ b/ydb/core/scheme_types/scheme_raw_type_value.h @@ -1,6 +1,6 @@ #pragma once -#include <ydb/public/lib/scheme_types/scheme_type_id.h> +#include "scheme_type_info.h" #include <util/generic/array_ref.h> #include <util/string/builder.h> @@ -17,23 +17,33 @@ public: , ValueType(0) {} - TRawTypeValue(TArrayRef<const char> ref, NScheme::TTypeId vtype) + TRawTypeValue(const void* buf, ui32 bufSize, NScheme::TTypeInfo vtype) + : Buffer(buf) + , BufferSize(bufSize) + , ValueType(vtype) + { + Y_VERIFY_DEBUG(!buf || vtype.GetTypeId() != 0); + } + + TRawTypeValue(TArrayRef<const char> ref, NScheme::TTypeInfo vtype) : TRawTypeValue((void*)ref.data(), ref.size(), vtype) { + } + TRawTypeValue(const void* buf, ui32 bufSize, NScheme::TTypeId typeId) + : TRawTypeValue(buf, bufSize, NScheme::TTypeInfo(typeId)) + { } - TRawTypeValue(const void* buf, ui32 bufSize, NScheme::TTypeId vtype) - : Buffer(buf) - , BufferSize(bufSize) - , ValueType(vtype) + TRawTypeValue(TArrayRef<const char> ref, NScheme::TTypeId typeId) + : TRawTypeValue((void*)ref.data(), ref.size(), NScheme::TTypeInfo(typeId)) { - Y_VERIFY_DEBUG(!buf || vtype); } const void* Data() const { return Buffer; } ui32 Size() const { return BufferSize; } - NScheme::TTypeId Type() const { return ValueType; } + NScheme::TTypeId Type() const { return ValueType.GetTypeId(); } + NScheme::TTypeInfo TypeInfo() const { return ValueType; } // we must distinguish empty raw type value (nothing, buffer == nullptr) // and zero-length string (value exists, but zero-length) @@ -42,7 +52,8 @@ public: TString ToString() const { TStringBuilder builder; - builder << "(type:" << ValueType; + // TODO: support pg types + builder << "(type:" << ValueType.GetTypeId(); if (!IsEmpty()) { builder << ", value:" << TString((const char*)Buffer, BufferSize).Quote(); } @@ -61,7 +72,7 @@ public: private: const void* Buffer; ui32 BufferSize; - NScheme::TTypeId ValueType; + NScheme::TTypeInfo ValueType; }; } // namspace NKikimr diff --git a/ydb/core/scheme_types/scheme_type_info.h b/ydb/core/scheme_types/scheme_type_info.h new file mode 100644 index 00000000000..86e8b1cedd1 --- /dev/null +++ b/ydb/core/scheme_types/scheme_type_info.h @@ -0,0 +1,45 @@ +#pragma once + +#include <ydb/public/lib/scheme_types/scheme_type_id.h> + +namespace NKikimr::NScheme { + +class TTypeInfo { +public: + constexpr TTypeInfo() + {} + + explicit constexpr TTypeInfo(TTypeId typeId, void* typeDesc = {}) + : TypeId(typeId) + , TypeDesc(typeDesc) + { + if (TypeId == NTypeIds::Pg) { + Y_VERIFY(TypeDesc); + } else { + Y_VERIFY(!TypeDesc); + } + } + + bool operator==(const TTypeInfo& other) const { + return TypeId == other.TypeId && TypeDesc == other.TypeDesc; + } + + bool operator!=(const TTypeInfo& other) const { + return !operator==(other); + } + + TTypeId GetTypeId() const { + return TypeId; + } + + void* GetTypeDesc() const { + return TypeDesc; + } + +private: + TTypeId TypeId = 0; + void* TypeDesc = {}; +}; + +} // namespace NKikimr::NScheme + diff --git a/ydb/core/scheme_types/scheme_types_defs.h b/ydb/core/scheme_types/scheme_types_defs.h index bb06b097023..590f218056e 100644 --- a/ydb/core/scheme_types/scheme_types_defs.h +++ b/ydb/core/scheme_types/scheme_types_defs.h @@ -49,7 +49,7 @@ public: } TTypeId GetTypeId() const override { return TypeId; } static TRawTypeValue ToRawTypeValue(const T& value) { - return TRawTypeValue((void*)&value, sizeof(T), TypeId); + return TRawTypeValue((void*)&value, sizeof(T), TTypeInfo(TypeId)); } static const char* TypeName() { @@ -151,7 +151,7 @@ public: } static TRawTypeValue ToRawTypeValue(const ::TString& value) { - return TRawTypeValue((const void*)value.data(), value.size(), TypeId); + return TRawTypeValue((const void*)value.data(), value.size(), TTypeInfo(TypeId)); } }; @@ -198,7 +198,7 @@ public: static TRawTypeValue ToRawTypeValue(const ::TString& value) { Y_VERIFY(value.size() <= MaxSize); - return TRawTypeValue((const void*)value.data(), value.size(), TypeId); + return TRawTypeValue((const void*)value.data(), value.size(), TTypeInfo(TypeId)); } }; diff --git a/ydb/core/security/ut/CMakeLists.darwin.txt b/ydb/core/security/ut/CMakeLists.darwin.txt index 52dfdab20d4..f2c4b7a21a5 100644 --- a/ydb/core/security/ut/CMakeLists.darwin.txt +++ b/ydb/core/security/ut/CMakeLists.darwin.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-core-security-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main ydb-core-security - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-security-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/security/ut/CMakeLists.linux.txt b/ydb/core/security/ut/CMakeLists.linux.txt index 28829eb9f01..83b99c49ad4 100644 --- a/ydb/core/security/ut/CMakeLists.linux.txt +++ b/ydb/core/security/ut/CMakeLists.linux.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-security-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main ydb-core-security - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-security-ut PRIVATE -ldl diff --git a/ydb/core/sys_view/common/schema.cpp b/ydb/core/sys_view/common/schema.cpp index 88f852e168c..78d7c1bf81a 100644 --- a/ydb/core/sys_view/common/schema.cpp +++ b/ydb/core/sys_view/common/schema.cpp @@ -114,7 +114,7 @@ private: static void Fill(TSchema& schema) { schema.Columns[Column::ColumnId] = TSysTables::TTableColumnInfo( Table::template TableColumns<Column>::GetColumnName(), - Column::ColumnId, Column::ColumnType, -1); + Column::ColumnId, NScheme::TTypeInfo(Column::ColumnType), -1); } }; diff --git a/ydb/core/sys_view/common/schema.h b/ydb/core/sys_view/common/schema.h index 77d330f2563..11954088d32 100644 --- a/ydb/core/sys_view/common/schema.h +++ b/ydb/core/sys_view/common/schema.h @@ -474,7 +474,7 @@ public: struct TSchema { THashMap<NTable::TTag, TSysTables::TTableColumnInfo> Columns; - TVector<NScheme::TTypeId> KeyColumnTypes; + TVector<NScheme::TTypeInfo> KeyColumnTypes; }; virtual bool IsSystemViewPath(const TVector<TString>& path, TSystemViewPath& sysViewPath) const = 0; diff --git a/ydb/core/sys_view/partition_stats/ut/CMakeLists.darwin.txt b/ydb/core/sys_view/partition_stats/ut/CMakeLists.darwin.txt index f82873c42b4..7a83cc82ab9 100644 --- a/ydb/core/sys_view/partition_stats/ut/CMakeLists.darwin.txt +++ b/ydb/core/sys_view/partition_stats/ut/CMakeLists.darwin.txt @@ -21,7 +21,7 @@ target_link_libraries(ydb-core-sys_view-partition_stats-ut PUBLIC cpp-testing-unittest_main core-sys_view-partition_stats cpp-testing-unittest - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-sys_view-partition_stats-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/sys_view/partition_stats/ut/CMakeLists.linux.txt b/ydb/core/sys_view/partition_stats/ut/CMakeLists.linux.txt index 133257735a1..d9bae3acf07 100644 --- a/ydb/core/sys_view/partition_stats/ut/CMakeLists.linux.txt +++ b/ydb/core/sys_view/partition_stats/ut/CMakeLists.linux.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-sys_view-partition_stats-ut PUBLIC cpp-testing-unittest_main core-sys_view-partition_stats cpp-testing-unittest - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-sys_view-partition_stats-ut PRIVATE -ldl diff --git a/ydb/core/sys_view/query_stats/ut/CMakeLists.darwin.txt b/ydb/core/sys_view/query_stats/ut/CMakeLists.darwin.txt index 3d0204c86ed..09fba86f255 100644 --- a/ydb/core/sys_view/query_stats/ut/CMakeLists.darwin.txt +++ b/ydb/core/sys_view/query_stats/ut/CMakeLists.darwin.txt @@ -21,7 +21,7 @@ target_link_libraries(ydb-core-sys_view-query_stats-ut PUBLIC cpp-testing-unittest_main core-sys_view-query_stats cpp-testing-unittest - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-sys_view-query_stats-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/sys_view/query_stats/ut/CMakeLists.linux.txt b/ydb/core/sys_view/query_stats/ut/CMakeLists.linux.txt index 6a4be7db09d..b0ad07f15a5 100644 --- a/ydb/core/sys_view/query_stats/ut/CMakeLists.linux.txt +++ b/ydb/core/sys_view/query_stats/ut/CMakeLists.linux.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-sys_view-query_stats-ut PUBLIC cpp-testing-unittest_main core-sys_view-query_stats cpp-testing-unittest - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-sys_view-query_stats-ut PRIVATE -ldl diff --git a/ydb/core/sys_view/storage/base.h b/ydb/core/sys_view/storage/base.h index 649d08482c1..47945cbbcf7 100644 --- a/ydb/core/sys_view/storage/base.h +++ b/ydb/core/sys_view/storage/base.h @@ -163,7 +163,7 @@ namespace NKikimr::NSysView { const NProtoBuf::Descriptor *desc = m->GetDescriptor(); const NProtoBuf::FieldDescriptor *fdesc = desc->FindFieldByNumber(*it); if (std::next(it) == path.end()) { // terminal entry - cells.push_back(ExtractCell(m, fdesc, column.Type)); + cells.push_back(ExtractCell(m, fdesc, column.Type.GetTypeId())); } else { // submessage Y_VERIFY(fdesc->type() == NProtoBuf::FieldDescriptor::TYPE_MESSAGE); m = &m->GetReflection()->GetMessage(*m, fdesc); diff --git a/ydb/core/sys_view/ut_kqp/CMakeLists.darwin.txt b/ydb/core/sys_view/ut_kqp/CMakeLists.darwin.txt index f4db6ac0686..7331e727891 100644 --- a/ydb/core/sys_view/ut_kqp/CMakeLists.darwin.txt +++ b/ydb/core/sys_view/ut_kqp/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-sys_view-ut_kqp PUBLIC cpp-testing-unittest cpp-yson-node kqp-ut-common - ydb-core-testlib + core-testlib-default cpp-client-draft ) target_link_options(ydb-core-sys_view-ut_kqp PRIVATE diff --git a/ydb/core/sys_view/ut_kqp/CMakeLists.linux.txt b/ydb/core/sys_view/ut_kqp/CMakeLists.linux.txt index 60cbd08188a..4e7d32c305e 100644 --- a/ydb/core/sys_view/ut_kqp/CMakeLists.linux.txt +++ b/ydb/core/sys_view/ut_kqp/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-sys_view-ut_kqp PUBLIC cpp-testing-unittest cpp-yson-node kqp-ut-common - ydb-core-testlib + core-testlib-default cpp-client-draft ) target_link_options(ydb-core-sys_view-ut_kqp PRIVATE diff --git a/ydb/core/tablet/ut/CMakeLists.darwin.txt b/ydb/core/tablet/ut/CMakeLists.darwin.txt index 6b9c7381921..c811e83b0de 100644 --- a/ydb/core/tablet/ut/CMakeLists.darwin.txt +++ b/ydb/core/tablet/ut/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tablet-ut PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-tablet-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/tablet/ut/CMakeLists.linux.txt b/ydb/core/tablet/ut/CMakeLists.linux.txt index 68814cb1c61..aae920bc0e5 100644 --- a/ydb/core/tablet/ut/CMakeLists.linux.txt +++ b/ydb/core/tablet/ut/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tablet-ut PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-tablet-ut PRIVATE -ldl diff --git a/ydb/core/tablet_flat/CMakeLists.txt b/ydb/core/tablet_flat/CMakeLists.txt index 05dd798c54b..3fe11bbf833 100644 --- a/ydb/core/tablet_flat/CMakeLists.txt +++ b/ydb/core/tablet_flat/CMakeLists.txt @@ -10,6 +10,7 @@ add_subdirectory(protos) add_subdirectory(test) add_subdirectory(ut) add_subdirectory(ut_large) +add_subdirectory(ut_pg) add_library(ydb-core-tablet_flat) target_compile_options(ydb-core-tablet_flat PRIVATE diff --git a/ydb/core/tablet_flat/flat_cxx_database.h b/ydb/core/tablet_flat/flat_cxx_database.h index dc3406f59c9..93f290712a3 100644 --- a/ydb/core/tablet_flat/flat_cxx_database.h +++ b/ydb/core/tablet_flat/flat_cxx_database.h @@ -2193,7 +2193,7 @@ struct TStaticSchemaFiller { schema.Columns[Column::ColumnId] = NTable::TColumn( TTable::template TableColumns<Column>::GetColumnName(), Column::ColumnId, - Column::ColumnType); + NScheme::TTypeInfo(Column::ColumnType)); } }; diff --git a/ydb/core/tablet_flat/flat_cxx_database_ut.cpp b/ydb/core/tablet_flat/flat_cxx_database_ut.cpp index 0423d82e624..876580b1af9 100644 --- a/ydb/core/tablet_flat/flat_cxx_database_ut.cpp +++ b/ydb/core/tablet_flat/flat_cxx_database_ut.cpp @@ -626,7 +626,7 @@ Y_UNIT_TEST_SUITE(TFlatCxxDatabaseTest) { for (const auto& col : columns) { ui32 id = schema.ColumnNames.at(col.first); UNIT_ASSERT_VALUES_EQUAL(schema.Columns.at(id).Name, col.first); - UNIT_ASSERT_VALUES_EQUAL(schema.Columns.at(id).PType, col.second); + UNIT_ASSERT_VALUES_EQUAL(schema.Columns.at(id).PType.GetTypeId(), col.second); } } } diff --git a/ydb/core/tablet_flat/flat_dbase_apply.cpp b/ydb/core/tablet_flat/flat_dbase_apply.cpp index 365286d4ddf..32ec4f02df6 100644 --- a/ydb/core/tablet_flat/flat_dbase_apply.cpp +++ b/ydb/core/tablet_flat/flat_dbase_apply.cpp @@ -32,8 +32,9 @@ bool TSchemeModifier::Apply(const TAlterRecord &delta) null = TCell(raw.data(), raw.size()); } - changes = AddColumn(table, delta.GetColumnName(), delta.GetColumnId(), - delta.GetColumnType(), delta.GetNotNull(), null); + ui32 pgTypeId = delta.HasColumnTypeInfo() ? delta.GetColumnTypeInfo().GetPgTypeId() : 0; + changes = AddPgColumn(table, delta.GetColumnName(), delta.GetColumnId(), + delta.GetColumnType(), pgTypeId, delta.GetNotNull(), null); } else if (action == TAlterRecord::DropColumn) { changes = DropColumn(table, delta.GetColumnId()); } else if (action == TAlterRecord::AddColumnToKey) { @@ -222,11 +223,27 @@ bool TSchemeModifier::DropTable(ui32 id) bool TSchemeModifier::AddColumn(ui32 tid, const TString &name, ui32 id, ui32 type, bool notNull, TCell null) { + Y_VERIFY(type != (ui32)NScheme::NTypeIds::Pg, "No pg type data"); + return AddPgColumn(tid, name, id, type, 0, notNull, null); +} + +bool TSchemeModifier::AddPgColumn(ui32 tid, const TString &name, ui32 id, ui32 type, ui32 pgType, bool notNull, TCell null) +{ auto *table = Table(tid); auto it = table->Columns.find(id); auto itName = table->ColumnNames.find(name); + NScheme::TTypeInfo typeInfo; + if (pgType != 0) { + Y_VERIFY((NScheme::TTypeId)type == NScheme::NTypeIds::Pg); + auto* typeDesc = NPg::TypeDescFromPgTypeId(pgType); + Y_VERIFY(typeDesc); + typeInfo = NScheme::TTypeInfo(type, typeDesc); + } else { + typeInfo = NScheme::TTypeInfo(type); + } + // We verify ids and types match when column with the same name already exists if (itName != table->ColumnNames.end()) { auto describeFailure = [&]() -> TString { @@ -242,8 +259,9 @@ bool TSchemeModifier::AddColumn(ui32 tid, const TString &name, ui32 id, ui32 typ Y_VERIFY_S(itName->second == id, describeFailure()); // Sanity check that this column exists and types match Y_VERIFY(it != table->Columns.end() && it->second.Name == name); - Y_VERIFY_S(it->second.PType == type, - "Table " << tid << " '" << table->Name << "' column " << id << " '" << name << "' expected type " << type << ", existing type " << it->second.PType); + Y_VERIFY_S(it->second.PType == typeInfo, + "Table " << tid << " '" << table->Name << "' column " << id << " '" << name + << "' expected type " << NScheme::TypeName(typeInfo) << ", existing type " << NScheme::TypeName(it->second.PType)); return false; } @@ -251,16 +269,16 @@ bool TSchemeModifier::AddColumn(ui32 tid, const TString &name, ui32 id, ui32 typ // We assume column is renamed when the same id already exists if (it != table->Columns.end()) { - Y_VERIFY_S(it->second.PType == type, + Y_VERIFY_S(it->second.PType == typeInfo, "Table " << tid << " '" << table->Name << "' column " << id << " '" << it->second.Name << "' renamed to '" << name << "'" - << " with type " << type << ", existing type " << it->second.PType); + << " with type " << NScheme::TypeName(typeInfo) << ", existing type " << NScheme::TypeName(it->second.PType)); table->ColumnNames.erase(it->second.Name); it->second.Name = name; table->ColumnNames.emplace(name, id); return true; } - auto pr = table->Columns.emplace(id, TColumn(name, id, type, notNull)); + auto pr = table->Columns.emplace(id, TColumn(name, id, typeInfo, notNull)); Y_VERIFY(pr.second); it = pr.first; table->ColumnNames.emplace(name, id); diff --git a/ydb/core/tablet_flat/flat_dbase_apply.h b/ydb/core/tablet_flat/flat_dbase_apply.h index 279a598c1f1..b9008b73abb 100644 --- a/ydb/core/tablet_flat/flat_dbase_apply.h +++ b/ydb/core/tablet_flat/flat_dbase_apply.h @@ -33,6 +33,7 @@ namespace NTable { bool AddTable(const TString& name, ui32 id); bool DropTable(ui32 id); bool AddColumn(ui32 table, const TString& name, ui32 id, ui32 type, bool notNull, TCell null = { }); + bool AddPgColumn(ui32 table, const TString& name, ui32 id, ui32 type, ui32 pgType, bool notNull, TCell null = { }); bool DropColumn(ui32 table, ui32 id); bool AddColumnToFamily(ui32 table, ui32 column, ui32 family); bool AddColumnToKey(ui32 table, ui32 column); diff --git a/ydb/core/tablet_flat/flat_dbase_scheme.cpp b/ydb/core/tablet_flat/flat_dbase_scheme.cpp index 61303df1049..2c973fa9044 100644 --- a/ydb/core/tablet_flat/flat_dbase_scheme.cpp +++ b/ydb/core/tablet_flat/flat_dbase_scheme.cpp @@ -28,7 +28,8 @@ TAutoPtr<TSchemeChanges> TScheme::GetSnapshot() const { for(const auto& it : itTable.second.Columns) { const auto &col = it.second; - delta.AddColumn(table, col.Name, it.first, col.PType, col.NotNull, col.Null); + delta.AddPgColumn(table, col.Name, it.first, col.PType.GetTypeId(), + NPg::PgTypeIdFromTypeDesc(col.PType.GetTypeDesc()), col.NotNull, col.Null); delta.AddColumnToFamily(table, it.first, col.Family); } @@ -97,12 +98,21 @@ TAlter& TAlter::DropTable(ui32 id) TAlter& TAlter::AddColumn(ui32 table, const TString& name, ui32 id, ui32 type, bool notNull, TCell null) { + Y_VERIFY(type != (ui32)NScheme::NTypeIds::Pg, "No pg type data"); + return AddPgColumn(table, name, id, type, 0, notNull, null); +} + +TAlter& TAlter::AddPgColumn(ui32 table, const TString& name, ui32 id, ui32 type, ui32 pgType, bool notNull, TCell null) +{ TAlterRecord& delta = *Log.AddDelta(); delta.SetDeltaType(TAlterRecord::AddColumn); delta.SetColumnName(name); delta.SetTableId(table); delta.SetColumnId(id); delta.SetColumnType(type); + if (pgType != 0) { + delta.MutableColumnTypeInfo()->SetPgTypeId(pgType); + } delta.SetNotNull(notNull); if (!null.IsNull()) diff --git a/ydb/core/tablet_flat/flat_dbase_scheme.h b/ydb/core/tablet_flat/flat_dbase_scheme.h index da4d0f7799f..56cc65ec94d 100644 --- a/ydb/core/tablet_flat/flat_dbase_scheme.h +++ b/ydb/core/tablet_flat/flat_dbase_scheme.h @@ -22,7 +22,6 @@ using TCompactionPolicy = NLocalDb::TCompactionPolicy; class TScheme { public: - using TTypeId = ui32; using ECache = NPage::ECache; enum EDefault { @@ -248,6 +247,7 @@ public: TAlter& AddTable(const TString& name, ui32 id); TAlter& DropTable(ui32 id); TAlter& AddColumn(ui32 table, const TString& name, ui32 id, ui32 type, bool notNull, TCell null = { }); + TAlter& AddPgColumn(ui32 table, const TString& name, ui32 id, ui32 type, ui32 pgType, bool notNull, TCell null = { }); TAlter& DropColumn(ui32 table, ui32 id); TAlter& AddColumnToFamily(ui32 table, ui32 column, ui32 family); TAlter& AddFamily(ui32 table, ui32 family, ui32 room); diff --git a/ydb/core/tablet_flat/flat_executor.cpp b/ydb/core/tablet_flat/flat_executor.cpp index 4e9719940a8..8e88e6ccd10 100644 --- a/ydb/core/tablet_flat/flat_executor.cpp +++ b/ydb/core/tablet_flat/flat_executor.cpp @@ -3977,7 +3977,7 @@ void TExecutor::RenderHtmlPage(NMon::TEvRemoteHttpInfo::TPtr &ev) const { TABLER() { TABLED() {str << col.Name;} TABLED() {str << col.Id;} - TABLED() {str << tr.GetTypeName(col.PType);} + TABLED() {str << tr.GetTypeName(col.PType.GetTypeId());} TABLED() {str << (isKey ? ToString(col.KeyOrder) : "");} } } diff --git a/ydb/core/tablet_flat/flat_executor_db_mon.cpp b/ydb/core/tablet_flat/flat_executor_db_mon.cpp index d5353747beb..50b8e7c32f9 100644 --- a/ydb/core/tablet_flat/flat_executor_db_mon.cpp +++ b/ydb/core/tablet_flat/flat_executor_db_mon.cpp @@ -78,7 +78,7 @@ public: break; } const auto& columnInfo = tableInfo->Columns.find(*itColumn)->second; - auto type = columnInfo.PType; + auto type = columnInfo.PType.GetTypeId(); switch (type) { case NScheme::NTypeIds::Uint32: { @@ -150,7 +150,7 @@ public: if (data == nullptr) { str << "<i><null></i>"; } else { - switch(tuple.Types[i]) { + switch(tuple.Types[i].GetTypeId()) { case NScheme::NTypeIds::Int8: str << *(i8*)data; break; @@ -221,8 +221,12 @@ public: str << "(DyNumber) " << number; break; } + case NScheme::NTypeIds::Pg: { + str << "(Pg) " << NPg::PgTypeNameFromTypeDesc(tuple.Types[i].GetTypeDesc()); + break; + } default: - str << "<i>unknown type " << tuple.Types[i] << "</i>"; + str << "<i>unknown type " << tuple.Types[i].GetTypeId() << "</i>"; break; } } diff --git a/ydb/core/tablet_flat/flat_executor_ut.cpp b/ydb/core/tablet_flat/flat_executor_ut.cpp index 2a3d9dcf90e..c5b3b298a88 100644 --- a/ydb/core/tablet_flat/flat_executor_ut.cpp +++ b/ydb/core/tablet_flat/flat_executor_ut.cpp @@ -2781,7 +2781,7 @@ Y_UNIT_TEST_SUITE(TFlatTableExecutorKeepEraseMarkers) { } TString value; - DbgPrintValue(value, row.Get(0), NScheme::TString::TypeId); + DbgPrintValue(value, row.Get(0), NScheme::TTypeInfo(NScheme::TString::TypeId)); builder << "Key " << keyId << " = " << row.GetRowState() << " value = " << NTable::ECellOp(row.GetCellOp(0)) << " " << value << Endl; @@ -3039,10 +3039,10 @@ Y_UNIT_TEST_SUITE(TFlatTableExecutorMoveTableData) { const auto& row = it->Row(); TString key; - DbgPrintValue(key, row.Get(0), NScheme::TUint64::TypeId); + DbgPrintValue(key, row.Get(0), NScheme::TTypeInfo(NScheme::TUint64::TypeId)); TString value; - DbgPrintValue(value, row.Get(1), NScheme::TString::TypeId); + DbgPrintValue(value, row.Get(1), NScheme::TTypeInfo(NScheme::TString::TypeId)); builder << "Key " << key << " = " << row.GetRowState() << " value = " << NTable::ECellOp(row.GetCellOp(1)) << " " << value << Endl; @@ -3352,10 +3352,10 @@ Y_UNIT_TEST_SUITE(TFlatTableExecutorFollower) { const auto& row = it->Row(); TString key; - DbgPrintValue(key, row.Get(0), NScheme::TUint64::TypeId); + DbgPrintValue(key, row.Get(0), NScheme::TTypeInfo(NScheme::TUint64::TypeId)); TString value; - DbgPrintValue(value, row.Get(1), NScheme::TString::TypeId); + DbgPrintValue(value, row.Get(1), NScheme::TTypeInfo(NScheme::TString::TypeId)); builder << "Key " << key << " = " << row.GetRowState() << " value = " << NTable::ECellOp(row.GetCellOp(1)) << " " << value << Endl; @@ -4129,13 +4129,13 @@ Y_UNIT_TEST_SUITE(TFlatTableLongTx) { const auto& row = it->Row(); TString key; - DbgPrintValue(key, row.Get(0), NScheme::TUint64::TypeId); + DbgPrintValue(key, row.Get(0), NScheme::TTypeInfo(NScheme::TUint64::TypeId)); TString value; - DbgPrintValue(value, row.Get(1), NScheme::TString::TypeId); + DbgPrintValue(value, row.Get(1), NScheme::TTypeInfo(NScheme::TString::TypeId)); TString value2; - DbgPrintValue(value2, row.Get(2), NScheme::TString::TypeId); + DbgPrintValue(value2, row.Get(2), NScheme::TTypeInfo(NScheme::TString::TypeId)); builder << "Key " << key << " = " << row.GetRowState() << " value = " << NTable::ECellOp(row.GetCellOp(1)) << " " << value @@ -4181,13 +4181,13 @@ Y_UNIT_TEST_SUITE(TFlatTableLongTx) { const auto& row = it->Row(); TString key; - DbgPrintValue(key, row.Get(0), NScheme::TUint64::TypeId); + DbgPrintValue(key, row.Get(0), NScheme::TTypeInfo(NScheme::TUint64::TypeId)); TString value; - DbgPrintValue(value, row.Get(1), NScheme::TString::TypeId); + DbgPrintValue(value, row.Get(1), NScheme::TTypeInfo(NScheme::TString::TypeId)); TString value2; - DbgPrintValue(value2, row.Get(2), NScheme::TString::TypeId); + DbgPrintValue(value2, row.Get(2), NScheme::TTypeInfo(NScheme::TString::TypeId)); builder << "Key " << key << " = " << row.GetRowState() << " value = " << NTable::ECellOp(row.GetCellOp(1)) << " " << value @@ -4230,13 +4230,13 @@ Y_UNIT_TEST_SUITE(TFlatTableLongTx) { const auto& row = it->Row(); TString key; - DbgPrintValue(key, row.Get(0), NScheme::TUint64::TypeId); + DbgPrintValue(key, row.Get(0), NScheme::TTypeInfo(NScheme::TUint64::TypeId)); TString value; - DbgPrintValue(value, row.Get(1), NScheme::TString::TypeId); + DbgPrintValue(value, row.Get(1), NScheme::TTypeInfo(NScheme::TString::TypeId)); TString value2; - DbgPrintValue(value2, row.Get(2), NScheme::TString::TypeId); + DbgPrintValue(value2, row.Get(2), NScheme::TTypeInfo(NScheme::TString::TypeId)); builder << "Key " << key << " = " << row.GetRowState() << " value = " << NTable::ECellOp(row.GetCellOp(1)) << " " << value diff --git a/ydb/core/tablet_flat/flat_executor_ut_large.cpp b/ydb/core/tablet_flat/flat_executor_ut_large.cpp index 0a163e6bc62..38e6359a131 100644 --- a/ydb/core/tablet_flat/flat_executor_ut_large.cpp +++ b/ydb/core/tablet_flat/flat_executor_ut_large.cpp @@ -122,13 +122,13 @@ Y_UNIT_TEST_SUITE(TFlatTableLongTxLarge) { const auto& row = it->Row(); TString key; - DbgPrintValue(key, row.Get(0), NScheme::TUint64::TypeId); + DbgPrintValue(key, row.Get(0), NScheme::TTypeInfo(NScheme::TUint64::TypeId)); TString value; - DbgPrintValue(value, row.Get(1), NScheme::TString::TypeId); + DbgPrintValue(value, row.Get(1), NScheme::TTypeInfo(NScheme::TString::TypeId)); TString value2; - DbgPrintValue(value2, row.Get(2), NScheme::TString::TypeId); + DbgPrintValue(value2, row.Get(2), NScheme::TTypeInfo(NScheme::TString::TypeId)); builder << "Key " << key << " = " << row.GetRowState() << " value = " << NTable::ECellOp(row.GetCellOp(1)) << " " << value diff --git a/ydb/core/tablet_flat/flat_iterator.h b/ydb/core/tablet_flat/flat_iterator.h index 9565082a30d..8ecd0cd4763 100644 --- a/ydb/core/tablet_flat/flat_iterator.h +++ b/ydb/core/tablet_flat/flat_iterator.h @@ -360,7 +360,7 @@ private: }; struct TComparator { - TComparator(TArrayRef<const NScheme::TTypeIdOrder> types) + TComparator(TArrayRef<const NScheme::TTypeInfoOrder> types) : Types(types) {} @@ -379,7 +379,7 @@ private: return a.IteratorId.Epoch < b.IteratorId.Epoch; } - const TArrayRef<const NScheme::TTypeIdOrder> Types; + const TArrayRef<const NScheme::TTypeInfoOrder> Types; }; static TIteratorIndex IteratorIndexFromSize(size_t size) { diff --git a/ydb/core/tablet_flat/flat_iterator_ops.h b/ydb/core/tablet_flat/flat_iterator_ops.h index fadddafccf4..67a6ccccb52 100644 --- a/ydb/core/tablet_flat/flat_iterator_ops.h +++ b/ydb/core/tablet_flat/flat_iterator_ops.h @@ -9,7 +9,7 @@ namespace NTable { struct TTableItOps { static inline int CompareKeys( - TArrayRef<const NScheme::TTypeIdOrder> types, + TArrayRef<const NScheme::TTypeInfoOrder> types, TArrayRef<const TCell> a, TArrayRef<const TCell> b) noexcept { @@ -158,7 +158,7 @@ struct TTableItOps { struct TTableItReverseOps { static int CompareKeys( - TArrayRef<const NScheme::TTypeIdOrder> types, + TArrayRef<const NScheme::TTypeInfoOrder> types, TArrayRef<const TCell> a, TArrayRef<const TCell> b) noexcept { diff --git a/ydb/core/tablet_flat/flat_mem_warm.cpp b/ydb/core/tablet_flat/flat_mem_warm.cpp index c4b4d5cac4b..bd59853e161 100644 --- a/ydb/core/tablet_flat/flat_mem_warm.cpp +++ b/ydb/core/tablet_flat/flat_mem_warm.cpp @@ -128,10 +128,10 @@ void TMemTable::DebugDump(IOutputStream& str, const NScheme::TTypeRegistry& type for (ui32 i = 0; i < row->Items; ++i) { TTag colId = row->Ops()[i].Tag; if (Scheme->ColInfo(colId)) { - NScheme::TTypeId typeId = Scheme->ColInfo(colId)->TypeId; + auto typeInfo = Scheme->ColInfo(colId)->TypeInfo; auto &op = row->Ops()[i]; - str << EOpToStr(ECellOp(op.Op)) << " " << op.Tag << " " << DbgPrintCell(op.Value, typeId, typeRegistry); + str << EOpToStr(ECellOp(op.Op)) << " " << op.Tag << " " << DbgPrintCell(op.Value, typeInfo, typeRegistry); } else { str << "unknown column " << colId; } diff --git a/ydb/core/tablet_flat/flat_mem_warm.h b/ydb/core/tablet_flat/flat_mem_warm.h index 4682a7ffcac..065bf486315 100644 --- a/ydb/core/tablet_flat/flat_mem_warm.h +++ b/ydb/core/tablet_flat/flat_mem_warm.h @@ -36,7 +36,7 @@ namespace NMem { }; struct TKeyCmp { - const NScheme::TTypeIdOrder* Types; + const NScheme::TTypeInfoOrder* Types; const ui32 Count; explicit TKeyCmp(const TRowScheme& rowScheme) @@ -362,9 +362,9 @@ namespace NMem { */ } else if (TCellOp::HaveNoPayload(ops[it].NormalizedCellOp())) { /* Payloadless ECellOp types may have zero type value */ - } else if (info->TypeId != ops[it].Value.Type()) { + } else if (info->TypeInfo.GetTypeId() != ops[it].Value.Type()) { Y_FAIL("Got an unexpected column type %" PRIu16 " in cell update for tag %" PRIu32 " (expected %" PRIu16 ")", - ops[it].Value.Type(), ops[it].Tag, info->TypeId); + ops[it].Value.Type(), ops[it].Tag, info->TypeInfo.GetTypeId()); } auto cell = ops[it].AsCell(); diff --git a/ydb/core/tablet_flat/flat_part_dump.cpp b/ydb/core/tablet_flat/flat_part_dump.cpp index f78a07e8dfd..44f7da969a4 100644 --- a/ydb/core/tablet_flat/flat_part_dump.cpp +++ b/ydb/core/tablet_flat/flat_part_dump.cpp @@ -103,7 +103,7 @@ namespace { for (auto off : xrange(part.Scheme->Groups[0].KeyTypes.size())) { Out << (off ? ", " : ""); - TName(part.Scheme->Groups[0].KeyTypes[off]); + TName(part.Scheme->Groups[0].KeyTypes[off].GetTypeId()); } Out << ")" << Endl; @@ -184,11 +184,11 @@ namespace { if (op == ELargeObj::Inline) { Out - << DbgPrintCell(iter->Cell(info), info.TypeId, *Reg); + << DbgPrintCell(iter->Cell(info), info.TypeInfo, *Reg); } else { const auto ref = iter->Cell(info).AsValue<ui64>(); - TName(info.TypeId); + TName(info.TypeInfo.GetTypeId()); const auto frame = GetFrame(part, ref, op); const auto blob = Env->Locate(&part, ref, op); diff --git a/ydb/core/tablet_flat/flat_part_scheme.h b/ydb/core/tablet_flat/flat_part_scheme.h index 543df5fcaca..f92f87827f6 100644 --- a/ydb/core/tablet_flat/flat_part_scheme.h +++ b/ydb/core/tablet_flat/flat_part_scheme.h @@ -41,7 +41,7 @@ namespace NTable { /* Data page key layout */ TVector<TColumn> ColsKeyData; - TVector<NScheme::TTypeId> KeyTypes; + TVector<NScheme::TTypeInfo> KeyTypes; /* Index page layout settings */ TPgSize IdxRecFixedSize; diff --git a/ydb/core/tablet_flat/flat_range_cache.h b/ydb/core/tablet_flat/flat_range_cache.h index 95fd100cdf4..c8046a85931 100644 --- a/ydb/core/tablet_flat/flat_range_cache.h +++ b/ydb/core/tablet_flat/flat_range_cache.h @@ -40,7 +40,7 @@ class TKeyRangeEntryCompare { public: typedef void is_transparent; - explicit TKeyRangeEntryCompare(TArrayRef<const NScheme::TTypeIdOrder> keyTypes) + explicit TKeyRangeEntryCompare(TArrayRef<const NScheme::TTypeInfoOrder> keyTypes) : KeyTypes(keyTypes) { } @@ -119,7 +119,7 @@ public: } private: - const TArrayRef<const NScheme::TTypeIdOrder> KeyTypes; + const TArrayRef<const NScheme::TTypeInfoOrder> KeyTypes; }; struct TKeyRangeEntryLRU diff --git a/ydb/core/tablet_flat/flat_range_cache_ut.cpp b/ydb/core/tablet_flat/flat_range_cache_ut.cpp index f518a8a1f74..2674e00212c 100644 --- a/ydb/core/tablet_flat/flat_range_cache_ut.cpp +++ b/ydb/core/tablet_flat/flat_range_cache_ut.cpp @@ -44,8 +44,8 @@ Y_UNIT_TEST_SUITE(TFlatEraseCacheTest) { */ size_t GetNodeSize() { // Construct fake TKeyCellDefaults - TVector<NScheme::TTypeIdOrder> types; - types.emplace_back(NScheme::NTypeIds::Uint64); + TVector<NScheme::TTypeInfoOrder> types; + types.emplace_back(NScheme::TTypeInfo(NScheme::NTypeIds::Uint64)); TVector<TCell> defs(1); TIntrusiveConstPtr<TKeyCellDefaults> keyDefaults = TKeyCellDefaults::Make(types, defs); // Empty default config diff --git a/ydb/core/tablet_flat/flat_row_column.h b/ydb/core/tablet_flat/flat_row_column.h index 391196d3f21..e6b1d403518 100644 --- a/ydb/core/tablet_flat/flat_row_column.h +++ b/ydb/core/tablet_flat/flat_row_column.h @@ -1,7 +1,7 @@ #pragma once #include "flat_row_eggs.h" -#include <ydb/core/scheme/scheme_type_id.h> +#include <ydb/core/scheme_types/scheme_type_info.h> namespace NKikimr { namespace NTable { @@ -42,7 +42,7 @@ namespace NTable { return Key != Max<TPos>(); } - NScheme::TTypeId TypeId = 0; + NScheme::TTypeInfo TypeInfo; TTag Tag = Max<TTag>(); TPos Pos = Max<TPos>(); /* Position in physical layout */ TPos Key = Max<TPos>(); /* key column sequence number */ diff --git a/ydb/core/tablet_flat/flat_row_misc.h b/ydb/core/tablet_flat/flat_row_misc.h index 4fbfe0de995..9eac844ccf6 100644 --- a/ydb/core/tablet_flat/flat_row_misc.h +++ b/ydb/core/tablet_flat/flat_row_misc.h @@ -13,7 +13,7 @@ namespace NFmt { struct TCells { using TReg = NScheme::TTypeRegistry; using TCellsRef = TArrayRef<const TCell>; - using TTypesRef = TArrayRef<const NScheme::TTypeId>; + using TTypesRef = TArrayRef<const NScheme::TTypeInfo>; TCells(TCellsRef cells, const NTable::TRemap &remap, const TReg *reg) : TCells(cells, remap.Types(), reg) diff --git a/ydb/core/tablet_flat/flat_row_nulls.h b/ydb/core/tablet_flat/flat_row_nulls.h index c53cf73aede..c731e4100c1 100644 --- a/ydb/core/tablet_flat/flat_row_nulls.h +++ b/ydb/core/tablet_flat/flat_row_nulls.h @@ -12,8 +12,8 @@ namespace NTable { class TCellDefaults: public TAtomicRefCount<TCellDefaults, NUtil::TDtorDel<TCellDefaults>> { protected: - using TType = NScheme::TTypeId; - using TOrder = NScheme::TTypeIdOrder; + using TType = NScheme::TTypeInfo; + using TOrder = NScheme::TTypeInfoOrder; TCellDefaults(TArrayRef<const TType> types, TArrayRef<const TCell> defs) : Types(types) @@ -154,7 +154,7 @@ namespace NTable { TStackVec<TType> types; types.reserve(order.size()); for (TOrder typeOrder : order) { - types.push_back(typeOrder.GetTypeId()); + types.push_back(NScheme::TTypeInfo(typeOrder.GetTypeId(), typeOrder.GetTypeDesc())); } return TCellDefaults::Make<TKeyCellDefaults>(types, order, defs); } diff --git a/ydb/core/tablet_flat/flat_row_remap.h b/ydb/core/tablet_flat/flat_row_remap.h index f948e82e3ac..e14e1c3bfa0 100644 --- a/ydb/core/tablet_flat/flat_row_remap.h +++ b/ydb/core/tablet_flat/flat_row_remap.h @@ -39,7 +39,7 @@ namespace NTable { TRemap(const TRowScheme& scheme, TTagsRef tags) : Tags(tags.begin(), tags.end()) - , Types_(tags.size(), 0) + , Types_(tags.size()) { Tag2Pos.reserve(tags.size()); CellDefaults_.reserve(tags.size()); @@ -47,7 +47,7 @@ namespace NTable { for (TPos on = 0; on < tags.size(); on++) { const auto *info = scheme.ColInfo(tags[on]); Y_VERIFY(info, "Column %" PRIu32 " does not exist", tags[on]); - Types_[on] = info->TypeId; + Types_[on] = info->TypeInfo; CellDefaults_.emplace_back((*scheme.RowCellDefaults)[info->Pos]); @@ -59,7 +59,7 @@ namespace NTable { } } - TArrayRef<const NScheme::TTypeId> Types() const noexcept + TArrayRef<const NScheme::TTypeInfo> Types() const noexcept { return Types_; } @@ -90,7 +90,7 @@ namespace NTable { const TSmallVec<TTag> Tags; private: TSmallVec<TPin> KeyPins_; - TSmallVec<NScheme::TTypeId> Types_; + TSmallVec<NScheme::TTypeInfo> Types_; TSmallVec<TCell> CellDefaults_; THashMap<TTag, TPos> Tag2Pos; }; diff --git a/ydb/core/tablet_flat/flat_row_scheme.h b/ydb/core/tablet_flat/flat_row_scheme.h index 4ee03bea873..19cc387d480 100644 --- a/ydb/core/tablet_flat/flat_row_scheme.h +++ b/ydb/core/tablet_flat/flat_row_scheme.h @@ -66,8 +66,8 @@ namespace NTable { keyCount += (meta.KeyOrder == Max<TPos>() ? 0 : 1); } - TNullsCook<TKeyCellDefaults, NScheme::TTypeIdOrder> keys(keyCount); - TNullsCook<TRowCellDefaults, NScheme::TTypeId> vals(cols.size()); + TNullsCook<TKeyCellDefaults, NScheme::TTypeInfoOrder> keys(keyCount); + TNullsCook<TRowCellDefaults, NScheme::TTypeInfo> vals(cols.size()); TVector<TColInfo> info; info.reserve(cols.size()); @@ -82,15 +82,15 @@ namespace NTable { Y_VERIFY(familyIt != families.end() && *familyIt == meta.Family); col.Tag = meta.Id; - col.TypeId = meta.PType; + col.TypeInfo = meta.PType; col.Key = meta.KeyOrder; col.Pos = info.size() - 1; col.Group = familyIt - families.begin(); - vals.Set(col.Pos, col.TypeId, meta.Null); + vals.Set(col.Pos, col.TypeInfo, meta.Null); if (col.IsKey()) - keys.Set(col.Key, col.TypeId, meta.Null); + keys.Set(col.Key, col.TypeInfo, meta.Null); } return new TRowScheme(std::move(info), *keys, *vals, std::move(families)); @@ -128,7 +128,7 @@ namespace NTable { Y_FAIL("Key column dropping ins't supported"); } else if (other == nullptr) { /* It is ok to drop non-key columns */ - } else if (col.TypeId != other->TypeId) { + } else if (col.TypeInfo != other->TypeInfo) { Y_FAIL("Column type alteration is not supproted"); } else if (col.Key != other->Key) { Y_FAIL("Cannot alter keys order or move col to keys"); @@ -142,8 +142,7 @@ namespace NTable { } else { auto &null = (*scheme.RowCellDefaults)[other->Pos]; - - if (CompareTypedCells(null, (*RowCellDefaults)[col.Pos], col.TypeId)) + if (CompareTypedCells(null, (*RowCellDefaults)[col.Pos], col.TypeInfo)) Y_FAIL("Cannot alter existing columnt default value"); } } diff --git a/ydb/core/tablet_flat/flat_table_column.h b/ydb/core/tablet_flat/flat_table_column.h index 2f9791a6363..49f8fb7a9d0 100644 --- a/ydb/core/tablet_flat/flat_table_column.h +++ b/ydb/core/tablet_flat/flat_table_column.h @@ -15,7 +15,7 @@ namespace NTable { TColumn() = default; - TColumn(const TString& name, TTag tag, NScheme::TTypeId type, bool notNull = false) + TColumn(const TString& name, TTag tag, NScheme::TTypeInfo type, bool notNull = false) : Id(tag) , PType(type) , Name(name) @@ -47,7 +47,7 @@ namespace NTable { } NTable::TTag Id = Max<TTag>(); - NScheme::TTypeId PType = 0; + NScheme::TTypeInfo PType; TString Name; ui32 Family = LeaderFamily; NTable::TPos KeyOrder = Max<TPos>(); diff --git a/ydb/core/tablet_flat/flat_table_part.cpp b/ydb/core/tablet_flat/flat_table_part.cpp index 0e55e1b42ea..ff042b266ff 100644 --- a/ydb/core/tablet_flat/flat_table_part.cpp +++ b/ydb/core/tablet_flat/flat_table_part.cpp @@ -4,6 +4,7 @@ #include "util_basics.h" #include <ydb/core/util/pb.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/tablet_flat/protos/flat_table_part.pb.h> #include <library/cpp/containers/stack_vector/stack_vec.h> #include <util/generic/map.h> @@ -52,7 +53,8 @@ TIntrusiveConstPtr<TPartScheme> TPartScheme::Parse(TArrayRef<const char> raw, bo cols.emplace_back(); cols.back().Tag = one.GetTag(); - cols.back().TypeId = one.GetType(); + cols.back().TypeInfo = NScheme::TypeInfoFromProtoColumnType(one.GetType(), + one.HasTypeInfo() ? &one.GetTypeInfo() : nullptr); cols.back().Pos = cols.size() - 1; cols.back().Group = one.GetGroup(); @@ -96,10 +98,10 @@ void TPartScheme::FillKeySlots() void TPartScheme::FillHistoricSlots() { // Synthetic (rowid, step, txid) key used during history searches - TStackVec<NScheme::TTypeIdOrder, 3> types; - types.emplace_back(NScheme::NTypeIds::Uint64, NScheme::EOrder::Ascending); - types.emplace_back(NScheme::NTypeIds::Uint64, NScheme::EOrder::Descending); - types.emplace_back(NScheme::NTypeIds::Uint64, NScheme::EOrder::Descending); + TStackVec<NScheme::TTypeInfoOrder, 3> types; + types.emplace_back(NScheme::TTypeInfo(NScheme::NTypeIds::Uint64), NScheme::EOrder::Ascending); + types.emplace_back(NScheme::TTypeInfo(NScheme::NTypeIds::Uint64), NScheme::EOrder::Descending); + types.emplace_back(NScheme::TTypeInfo(NScheme::NTypeIds::Uint64), NScheme::EOrder::Descending); TStackVec<TCell, 3> defs; defs.resize(3); HistoryKeys = TKeyCellDefaults::Make(types, defs); @@ -109,7 +111,7 @@ void TPartScheme::FillHistoricSlots() for (ui32 keyIdx = 0; keyIdx < HistoryKeys->Types.size(); ++keyIdx) { auto& col = HistoryGroup.Columns.emplace_back(); col.Key = keyIdx; - col.TypeId = HistoryKeys->Types[keyIdx].GetTypeId(); + col.TypeInfo = HistoryKeys->Types[keyIdx].ToTypeInfo(); } // All non-key columns go after synthetic key @@ -157,7 +159,7 @@ void TPartScheme::InitGroup(TGroupInfo& group) std::sort(group.ColsKeyData.begin(), group.ColsKeyData.end(), byKey); for (auto& col : group.ColsKeyData) { - group.KeyTypes.push_back(col.TypeId); + group.KeyTypes.push_back(col.TypeInfo); } group.ColsKeyIdx = group.ColsKeyData; @@ -172,7 +174,7 @@ size_t TPartScheme::InitInfo(TVector<TColumn>& cols, TPgSize headerSize) size_t offset = 0; for (auto &col: cols) { - const ui32 fixed = NScheme::GetFixedSize(col.TypeId); + const ui32 fixed = NScheme::GetFixedSize(col.TypeInfo); col.Offset = offset; col.IsFixed = fixed > 0; @@ -191,7 +193,11 @@ TSharedData TPartScheme::Serialize() const for (const auto& col : AllColumns) { auto* pb = proto.AddColumns(); pb->SetTag(col.Tag); - pb->SetType(col.TypeId); + auto protoType = NScheme::ProtoColumnTypeFromTypeInfo(col.TypeInfo); + pb->SetType(protoType.TypeId); + if (protoType.TypeInfo) { + *pb->MutableTypeInfo() = *protoType.TypeInfo; + } pb->SetGroup(col.Group); if (col.IsKey()) { diff --git a/ydb/core/tablet_flat/protos/flat_table_part.proto b/ydb/core/tablet_flat/protos/flat_table_part.proto index 61fa39e9573..53892fabfe0 100644 --- a/ydb/core/tablet_flat/protos/flat_table_part.proto +++ b/ydb/core/tablet_flat/protos/flat_table_part.proto @@ -1,4 +1,5 @@ import "ydb/core/protos/base.proto"; +import "ydb/core/protos/type_info.proto"; package NKikimr.NTable.NProto; option java_package = "ru.yandex.kikimr.proto"; @@ -11,6 +12,7 @@ message TEvol { message TPartColumn { optional uint32 Tag = 1; optional uint32 Type = 2; + optional NKikimrProto.TTypeInfo TypeInfo = 5; optional uint32 Key = 3; optional uint32 Group = 4; } diff --git a/ydb/core/tablet_flat/test/libs/rows/all.cpp b/ydb/core/tablet_flat/test/libs/rows/all.cpp index 796b82e0031..a6e0c8b0096 100644 --- a/ydb/core/tablet_flat/test/libs/rows/all.cpp +++ b/ydb/core/tablet_flat/test/libs/rows/all.cpp @@ -19,7 +19,7 @@ const NScheme::TTypeRegistry* DbgRegistry() TString PrintValue(const TCell& r, NScheme::TTypeId typeId) { - return DbgPrintCell(r, typeId, TypeRegistry); + return DbgPrintCell(r, NScheme::TTypeInfo(typeId), TypeRegistry); } } diff --git a/ydb/core/tablet_flat/test/libs/rows/layout.h b/ydb/core/tablet_flat/test/libs/rows/layout.h index 36e64feac08..6657b4fd08a 100644 --- a/ydb/core/tablet_flat/test/libs/rows/layout.h +++ b/ydb/core/tablet_flat/test/libs/rows/layout.h @@ -22,7 +22,8 @@ namespace NTest{ { Tags_.push_back(tag); - Cols.emplace_back("", tag, type); + // pg types are not supported + Cols.emplace_back("", tag, NScheme::TTypeInfo(type)); Cols.back().Family = group; Cols.back().SetDefault(null); diff --git a/ydb/core/tablet_flat/test/libs/rows/tool.h b/ydb/core/tablet_flat/test/libs/rows/tool.h index 18485858665..ac746d3702c 100644 --- a/ydb/core/tablet_flat/test/libs/rows/tool.h +++ b/ydb/core/tablet_flat/test/libs/rows/tool.h @@ -123,7 +123,7 @@ namespace NTest { } else if (val.Op == ECellOp::Empty || val.Op == ECellOp::Reset) { if (ECellOp(val.Op) != state.GetCellOp(pos)) return false; - } else if (CompareTypedCells(val.Cell, cell, info->TypeId)) { + } else if (CompareTypedCells(val.Cell, cell, info->TypeInfo)) { return false; /* Literal comparison has been failed */ } @@ -172,11 +172,11 @@ namespace NTest { } else if (value.Cell.IsNull() && value.Type == 0) { /* null values should be passed without type */ - } else if (info->TypeId != value.Type) { + } else if (info->TypeInfo.GetTypeId() != value.Type) { ythrow yexception() << "Col{" << info->Pos << ", Tag " << info->Tag - << ", type " << info->TypeId << "}" + << ", type " << info->TypeInfo.GetTypeId() << "}" << ", got type " << value.Type << " { " << PrintValue(value.Cell, value.Type) << " }"; } diff --git a/ydb/core/tablet_flat/ut/CMakeLists.darwin.txt b/ydb/core/tablet_flat/ut/CMakeLists.darwin.txt index de5d414efea..2d562ed938e 100644 --- a/ydb/core/tablet_flat/ut/CMakeLists.darwin.txt +++ b/ydb/core/tablet_flat/ut/CMakeLists.darwin.txt @@ -21,7 +21,7 @@ target_link_libraries(ydb-core-tablet_flat-ut PUBLIC ydb-core-scheme test-libs-exec test-libs-table - ydb-core-testlib + core-testlib-default udf-service-exception_policy ) target_link_options(ydb-core-tablet_flat-ut PRIVATE diff --git a/ydb/core/tablet_flat/ut/CMakeLists.linux.txt b/ydb/core/tablet_flat/ut/CMakeLists.linux.txt index 846c4ab036a..d1cf46b8bf1 100644 --- a/ydb/core/tablet_flat/ut/CMakeLists.linux.txt +++ b/ydb/core/tablet_flat/ut/CMakeLists.linux.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tablet_flat-ut PUBLIC ydb-core-scheme test-libs-exec test-libs-table - ydb-core-testlib + core-testlib-default udf-service-exception_policy ) target_link_options(ydb-core-tablet_flat-ut PRIVATE diff --git a/ydb/core/tablet_flat/ut/flat_test_db.cpp b/ydb/core/tablet_flat/ut/flat_test_db.cpp index b65757d4bbd..31d950ab36a 100644 --- a/ydb/core/tablet_flat/ut/flat_test_db.cpp +++ b/ydb/core/tablet_flat/ut/flat_test_db.cpp @@ -26,7 +26,8 @@ struct TFakeKeyComparator { if (bv.IsEmpty()) return false; Y_ASSERT(av.Type() == bv.Type()); - int res = CompareTypedCells(TCell(&av), TCell(&bv), av.Type()); + // pg types are not supported + int res = CompareTypedCells(TCell(&av), TCell(&bv), NScheme::TTypeInfo(av.Type(), {})); if (res) return res < 0; } @@ -76,10 +77,10 @@ class TFakeDbIterator : public ITestIterator { TFakeTable::const_iterator RowIt; TFakeTable::const_iterator RowEnd; - TVector<NScheme::TTypeId> KeyTypes; + TVector<NScheme::TTypeInfo> KeyTypes; TVector<TCell> KeyCells; - TVector<NScheme::TTypeId> ValueTypes; + TVector<NScheme::TTypeInfo> ValueTypes; TVector<TCell> ValueCells; bool First = true; diff --git a/ydb/core/tablet_flat/ut/flat_test_db_helpers.h b/ydb/core/tablet_flat/ut/flat_test_db_helpers.h index d75e08e3867..7c8c87159b9 100644 --- a/ydb/core/tablet_flat/ut/flat_test_db_helpers.h +++ b/ydb/core/tablet_flat/ut/flat_test_db_helpers.h @@ -146,7 +146,7 @@ public: Y_VERIFY(tagId, "Unknown column \"%s\" in table %u", tagName.data(), GetRoot()); const auto *colInfo = Scheme.GetColumnInfo(GetRoot(), *tagId); Y_VERIFY(colInfo, "Column info not found for table id %u, column id %u", GetRoot(), *tagId); - NScheme::TTypeId type = colInfo->PType; + NScheme::TTypeId type = colInfo->PType.GetTypeId(); TagOps[*tagId] = FromVal(type, val); return *this; } @@ -180,7 +180,7 @@ template <typename T, typename... Tt> void AppendKeyColumn(ui32 root, const TScheme& scheme, TVector<TFakeTableCell>& tuple, T t, Tt... tt) { ui32 pos = tuple.size(); ui32 tag = scheme.GetTableInfo(root)->KeyColumns[pos]; - NScheme::TTypeId type = scheme.GetColumnInfo(root, tag)->PType; + NScheme::TTypeId type = scheme.GetColumnInfo(root, tag)->PType.GetTypeId(); tuple.push_back(FromVal(type, t)); AppendKeyColumn(root, scheme, tuple, tt...); } diff --git a/ydb/core/tablet_flat/ut/ut_db_scheme.cpp b/ydb/core/tablet_flat/ut/ut_db_scheme.cpp index ee12efa9032..981c77243d5 100644 --- a/ydb/core/tablet_flat/ut/ut_db_scheme.cpp +++ b/ydb/core/tablet_flat/ut/ut_db_scheme.cpp @@ -41,8 +41,8 @@ namespace { bld.AddFamily(TableId, GroupId1, StoreIdDef); bld.SetFamily(TableId, GroupId1, ECache::Ever, ECodec::LZ4); bld.AddFamily(TableId, GroupId2, StoreIdOut); - bld.AddColumn(TableId, "key", ColId1, NScheme::TSmallBoundedString::TypeId, { }); - bld.AddColumn(TableId, "value", ColId2, NScheme::TUint32::TypeId, { }); + bld.AddColumn(TableId, "key", ColId1, NScheme::TSmallBoundedString::TypeId, false); + bld.AddColumn(TableId, "value", ColId2, NScheme::TUint32::TypeId, false); bld.AddColumnToKey(TableId, ColId1); bld.AddColumnToFamily(TableId, ColId1, GroupId1); bld.AddColumnToFamily(TableId, ColId2, GroupId2); diff --git a/ydb/core/tablet_flat/ut_pg/CMakeLists.darwin.txt b/ydb/core/tablet_flat/ut_pg/CMakeLists.darwin.txt new file mode 100644 index 00000000000..38b07eaf3fc --- /dev/null +++ b/ydb/core/tablet_flat/ut_pg/CMakeLists.darwin.txt @@ -0,0 +1,52 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-core-tablet_flat-ut_pg) +target_compile_options(ydb-core-tablet_flat-ut_pg PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION + -Wno-everything +) +target_include_directories(ydb-core-tablet_flat-ut_pg PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat + ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/include +) +target_link_libraries(ydb-core-tablet_flat-ut_pg PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + ydb-core-tablet_flat + ydb-core-scheme + test-libs-table + udf-service-exception_policy + yql-sql-pg +) +target_link_options(ydb-core-tablet_flat-ut_pg PRIVATE + -Wl,-no_deduplicate + -Wl,-sdk_version,10.15 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-core-tablet_flat-ut_pg PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut_pg/flat_database_pg_ut.cpp +) +add_test( + NAME + ydb-core-tablet_flat-ut_pg + COMMAND + ydb-core-tablet_flat-ut_pg + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +vcs_info(ydb-core-tablet_flat-ut_pg) diff --git a/ydb/core/tablet_flat/ut_pg/CMakeLists.linux.txt b/ydb/core/tablet_flat/ut_pg/CMakeLists.linux.txt new file mode 100644 index 00000000000..bb90d54f095 --- /dev/null +++ b/ydb/core/tablet_flat/ut_pg/CMakeLists.linux.txt @@ -0,0 +1,56 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-core-tablet_flat-ut_pg) +target_compile_options(ydb-core-tablet_flat-ut_pg PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION + -Wno-everything +) +target_include_directories(ydb-core-tablet_flat-ut_pg PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat + ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/include +) +target_link_libraries(ydb-core-tablet_flat-ut_pg PUBLIC + contrib-libs-cxxsupp + yutil + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache + library-cpp-cpuid_check + cpp-testing-unittest_main + ydb-core-tablet_flat + ydb-core-scheme + test-libs-table + udf-service-exception_policy + yql-sql-pg +) +target_link_options(ydb-core-tablet_flat-ut_pg PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-core-tablet_flat-ut_pg PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/ut_pg/flat_database_pg_ut.cpp +) +add_test( + NAME + ydb-core-tablet_flat-ut_pg + COMMAND + ydb-core-tablet_flat-ut_pg + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +vcs_info(ydb-core-tablet_flat-ut_pg) diff --git a/ydb/core/tablet_flat/ut_pg/CMakeLists.txt b/ydb/core/tablet_flat/ut_pg/CMakeLists.txt new file mode 100644 index 00000000000..79468a5d8d0 --- /dev/null +++ b/ydb/core/tablet_flat/ut_pg/CMakeLists.txt @@ -0,0 +1,13 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (APPLE) + include(CMakeLists.darwin.txt) +elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND UNIX AND NOT APPLE AND NOT ANDROID) + include(CMakeLists.linux.txt) +endif() diff --git a/ydb/core/tablet_flat/ut_pg/flat_database_pg_ut.cpp b/ydb/core/tablet_flat/ut_pg/flat_database_pg_ut.cpp new file mode 100644 index 00000000000..4abd5e70191 --- /dev/null +++ b/ydb/core/tablet_flat/ut_pg/flat_database_pg_ut.cpp @@ -0,0 +1,142 @@ +#include <library/cpp/testing/unittest/registar.h> +#include <ydb/core/tablet_flat/test/libs/table/test_dbase.h> + +extern "C" { +#include "postgres.h" +#include "catalog/pg_type_d.h" +} + +namespace NKikimr { +namespace NTable { + +Y_UNIT_TEST_SUITE(TFlatDatabasePgTest) { + + Y_UNIT_TEST(BasicTypes) { + constexpr ui32 tableId = 1; + + enum EColumnIds : ui32 { + IdBool = 1, + IdChar = 2, + IdInt2 = 3, + IdInt4 = 4, + IdInt8 = 5, + IdFloat4 = 6, + IdFloat8 = 7, + IdText = 8, + IdBytea = 9, + IdBpchar = 10, + }; + + NTest::TDbExec db; + + auto makePgType = [] (ui32 oid) { + return NScheme::TTypeInfo(NScheme::NTypeIds::Pg, NPg::TypeDescFromPgTypeId(oid)); + }; + + db.Begin(); + db->Alter() + .AddTable("TestTable", tableId) + .AddPgColumn(tableId, "boolean", IdBool, NScheme::NTypeIds::Pg, BOOLOID, false) + .AddPgColumn(tableId, "char", IdChar, NScheme::NTypeIds::Pg, CHAROID, false) + .AddPgColumn(tableId, "int2", IdInt2, NScheme::NTypeIds::Pg, INT2OID, false) + .AddPgColumn(tableId, "int4", IdInt4, NScheme::NTypeIds::Pg, INT4OID, false) + .AddPgColumn(tableId, "int8", IdInt8, NScheme::NTypeIds::Pg, INT8OID, false) + .AddPgColumn(tableId, "float4", IdFloat4, NScheme::NTypeIds::Pg, FLOAT4OID, false) + .AddPgColumn(tableId, "float8", IdFloat8, NScheme::NTypeIds::Pg, FLOAT8OID, false) + .AddPgColumn(tableId, "text", IdText, NScheme::NTypeIds::Pg, TEXTOID, false) + .AddPgColumn(tableId, "bytea", IdBytea, NScheme::NTypeIds::Pg, BYTEAOID, false) + .AddPgColumn(tableId, "bpchar", IdBpchar, NScheme::NTypeIds::Pg, BPCHAROID, false) + .AddColumnToKey(tableId, IdText) + .AddColumnToKey(tableId, IdBytea) + .AddColumnToKey(tableId, IdInt2) + .AddColumnToKey(tableId, IdFloat4); + db.Commit(); + + bool boolVal = true; + char charVal = 'a'; + i16 i16Val = 2; + i32 i32Val = 3; + i64 i64Val = 4; + float floatVal = 5.f; + double doubleVal = 6.; + auto strText = TString("abc"); + auto strBytea = TString("bytea"); + auto strBpchar = TString("bpchar"); + + for (int i = 1; i <= 10; ++i) { + db.Begin(); + + TVector<TRawTypeValue> key; + key.emplace_back(strText.data(), strText.size(), makePgType(TEXTOID)); + key.emplace_back(strBytea.data(), strBytea.size(), makePgType(BYTEAOID)); + key.emplace_back(&i16Val, sizeof(i16), makePgType(INT2OID)); + float f = (float)i; + key.emplace_back(&f, sizeof(float), makePgType(FLOAT4OID)); + + TVector<NTable::TUpdateOp> ops; + ops.emplace_back(IdBool, NTable::ECellOp::Set, + TRawTypeValue(&boolVal, sizeof(bool), makePgType(BOOLOID))); + ops.emplace_back(IdChar, NTable::ECellOp::Set, + TRawTypeValue(&charVal, sizeof(char), makePgType(CHAROID))); + ops.emplace_back(IdInt4, NTable::ECellOp::Set, + TRawTypeValue(&i32Val, sizeof(i32), makePgType(INT4OID))); + ops.emplace_back(IdInt8, NTable::ECellOp::Set, + TRawTypeValue(&i64Val, sizeof(i64), makePgType(INT8OID))); + ops.emplace_back(IdFloat8, NTable::ECellOp::Set, + TRawTypeValue(&doubleVal, sizeof(double), makePgType(FLOAT8OID))); + ops.emplace_back(IdBpchar, NTable::ECellOp::Set, + TRawTypeValue(strBpchar.data(), strBpchar.size(), makePgType(BPCHAROID))); + + db->Update(tableId, NTable::ERowOp::Upsert, key, ops); + + db.Commit(); + } + + TVector<NTable::TTag> tags; + for (NTable::TTag t = 1; t <= 10; ++t) { + tags.push_back(t); + } + + auto readDatabase = [&] () { + db.Begin(); + + TVector<TRawTypeValue> key; + key.emplace_back(strText.data(), strText.size(), makePgType(TEXTOID)); + key.emplace_back(strBytea.data(), strBytea.size(), makePgType(BYTEAOID)); + key.emplace_back(&i16Val, sizeof(i16), makePgType(INT2OID)); + key.emplace_back(&floatVal, sizeof(float), makePgType(FLOAT4OID)); + + auto it = db->Iterate(tableId, key, tags, ELookup::GreaterThan); + size_t count = 0; + while (it->Next(NTable::ENext::All) == NTable::EReady::Data) { + auto key = it->GetKey(); + auto value = it->GetValues(); + UNIT_ASSERT_VALUES_EQUAL(key.ColumnCount, 4); + UNIT_ASSERT_VALUES_EQUAL(value.ColumnCount, 10); + + UNIT_ASSERT_VALUES_EQUAL(value.Columns[0].AsValue<bool>(), boolVal); + UNIT_ASSERT_VALUES_EQUAL(value.Columns[1].AsValue<char>(), charVal); + UNIT_ASSERT_VALUES_EQUAL(value.Columns[2].AsValue<i16>(), i16Val); + UNIT_ASSERT_VALUES_EQUAL(value.Columns[3].AsValue<i32>(), i32Val); + UNIT_ASSERT_VALUES_EQUAL(value.Columns[4].AsValue<i64>(), i64Val); + UNIT_ASSERT(value.Columns[5].AsValue<float>() > floatVal); + UNIT_ASSERT_VALUES_EQUAL(value.Columns[6].AsValue<double>(), doubleVal); + UNIT_ASSERT(std::memcmp(value.Columns[7].Data(), strText.data(), value.Columns[7].Size()) == 0); + UNIT_ASSERT(std::memcmp(value.Columns[8].Data(), strBytea.data(), value.Columns[8].Size()) == 0); + UNIT_ASSERT(std::memcmp(value.Columns[9].Data(), strBpchar.data(), value.Columns[9].Size()) == 0); + ++count; + } + + db.Commit(); + }; + + readDatabase(); + + db.Snap(tableId).Compact(tableId, false); + + readDatabase(); + } +} + +} +} diff --git a/ydb/core/tablet_flat/util_fmt_cell.h b/ydb/core/tablet_flat/util_fmt_cell.h index d1f647a3ddf..f530b06d360 100644 --- a/ydb/core/tablet_flat/util_fmt_cell.h +++ b/ydb/core/tablet_flat/util_fmt_cell.h @@ -9,7 +9,7 @@ namespace NFmt { class TPrintableTypedCells { public: - TPrintableTypedCells(TArrayRef<const TCell> cells, TArrayRef<const NScheme::TTypeId> types) + TPrintableTypedCells(TArrayRef<const TCell> cells, TArrayRef<const NScheme::TTypeInfo> types) : Cells(cells) , Types(types) { @@ -33,7 +33,7 @@ public: private: const TArrayRef<const TCell> Cells; - const TArrayRef<const NScheme::TTypeId> Types; + const TArrayRef<const NScheme::TTypeInfo> Types; }; } diff --git a/ydb/core/testlib/CMakeLists.txt b/ydb/core/testlib/CMakeLists.txt index 104f9124f7b..98c3ba82f14 100644 --- a/ydb/core/testlib/CMakeLists.txt +++ b/ydb/core/testlib/CMakeLists.txt @@ -8,6 +8,8 @@ add_subdirectory(actors) add_subdirectory(basics) +add_subdirectory(default) +add_subdirectory(pg) add_library(ydb-core-testlib) target_compile_options(ydb-core-testlib PRIVATE diff --git a/ydb/core/testlib/basics/CMakeLists.txt b/ydb/core/testlib/basics/CMakeLists.txt index c968c145732..4703caf15ee 100644 --- a/ydb/core/testlib/basics/CMakeLists.txt +++ b/ydb/core/testlib/basics/CMakeLists.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(default) add_library(core-testlib-basics) target_compile_options(core-testlib-basics PRIVATE @@ -34,7 +35,6 @@ target_link_libraries(core-testlib-basics PUBLIC ydb-core-util yql-minikql-invoke_builtins udf-service-exception_policy - yql-sql-pg_dummy ) target_sources(core-testlib-basics PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/testlib/basics/appdata.cpp diff --git a/ydb/core/testlib/basics/default/CMakeLists.txt b/ydb/core/testlib/basics/default/CMakeLists.txt new file mode 100644 index 00000000000..6339cd1b4b7 --- /dev/null +++ b/ydb/core/testlib/basics/default/CMakeLists.txt @@ -0,0 +1,16 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(testlib-basics-default INTERFACE) +target_link_libraries(testlib-basics-default INTERFACE + contrib-libs-cxxsupp + yutil + core-testlib-basics + yql-sql-pg_dummy +) diff --git a/ydb/core/testlib/default/CMakeLists.txt b/ydb/core/testlib/default/CMakeLists.txt new file mode 100644 index 00000000000..7f271a88a68 --- /dev/null +++ b/ydb/core/testlib/default/CMakeLists.txt @@ -0,0 +1,16 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(core-testlib-default INTERFACE) +target_link_libraries(core-testlib-default INTERFACE + contrib-libs-cxxsupp + yutil + ydb-core-testlib + yql-sql-pg_dummy +) diff --git a/ydb/core/testlib/pg/CMakeLists.txt b/ydb/core/testlib/pg/CMakeLists.txt new file mode 100644 index 00000000000..262e65a2606 --- /dev/null +++ b/ydb/core/testlib/pg/CMakeLists.txt @@ -0,0 +1,16 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(core-testlib-pg INTERFACE) +target_link_libraries(core-testlib-pg INTERFACE + contrib-libs-cxxsupp + yutil + ydb-core-testlib + yql-sql-pg +) diff --git a/ydb/core/tx/balance_coverage/ut/CMakeLists.darwin.txt b/ydb/core/tx/balance_coverage/ut/CMakeLists.darwin.txt index b4f3bbee631..8bca67bafb8 100644 --- a/ydb/core/tx/balance_coverage/ut/CMakeLists.darwin.txt +++ b/ydb/core/tx/balance_coverage/ut/CMakeLists.darwin.txt @@ -18,7 +18,7 @@ target_link_libraries(ydb-core-tx-balance_coverage-ut PUBLIC cpp-testing-unittest_main core-tx-balance_coverage cpp-testing-unittest - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-tx-balance_coverage-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/tx/balance_coverage/ut/CMakeLists.linux.txt b/ydb/core/tx/balance_coverage/ut/CMakeLists.linux.txt index bd6824df917..540643cb5e7 100644 --- a/ydb/core/tx/balance_coverage/ut/CMakeLists.linux.txt +++ b/ydb/core/tx/balance_coverage/ut/CMakeLists.linux.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-core-tx-balance_coverage-ut PUBLIC cpp-testing-unittest_main core-tx-balance_coverage cpp-testing-unittest - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-tx-balance_coverage-ut PRIVATE -ldl diff --git a/ydb/core/tx/columnshard/columnshard__scan.cpp b/ydb/core/tx/columnshard/columnshard__scan.cpp index 01a26bc26f6..41245654f76 100644 --- a/ydb/core/tx/columnshard/columnshard__scan.cpp +++ b/ydb/core/tx/columnshard/columnshard__scan.cpp @@ -474,8 +474,8 @@ private: ui32 ReadMetadataIndex; std::unique_ptr<TScanIteratorBase> ScanIterator; - TVector<std::pair<TString, NScheme::TTypeId>> ResultYqlSchema; - TVector<std::pair<TString, NScheme::TTypeId>> KeyYqlSchema; + TVector<std::pair<TString, NScheme::TTypeInfo>> ResultYqlSchema; + TVector<std::pair<TString, NScheme::TTypeInfo>> KeyYqlSchema; const TSerializedTableRange TableRange; const TSmallVec<bool> SkipNullKeys; const TInstant Deadline; @@ -496,7 +496,7 @@ private: }; static void FillPredicatesFromRange(TReadDescription& read, const ::NKikimrTx::TKeyRange& keyRange, - const TVector<std::pair<TString, NScheme::TTypeId>>& ydbPk, ui64 tabletId) { + const TVector<std::pair<TString, NScheme::TTypeInfo>>& ydbPk, ui64 tabletId) { TSerializedTableRange range(keyRange); read.GreaterPredicate = std::make_shared<NOlap::TPredicate>(); diff --git a/ydb/core/tx/columnshard/columnshard_common.cpp b/ydb/core/tx/columnshard/columnshard_common.cpp index 25337a206fe..7c81aabe334 100644 --- a/ydb/core/tx/columnshard/columnshard_common.cpp +++ b/ydb/core/tx/columnshard/columnshard_common.cpp @@ -6,8 +6,8 @@ namespace NKikimr::NColumnShard { namespace { -TVector<NScheme::TTypeId> ExtractTypes(const TVector<std::pair<TString, NScheme::TTypeId>>& columns) { - TVector<NScheme::TTypeId> types; +TVector<NScheme::TTypeInfo> ExtractTypes(const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns) { + TVector<NScheme::TTypeInfo> types; types.reserve(columns.size()); for (auto& [name, type] : columns) { types.push_back(type); @@ -15,13 +15,13 @@ TVector<NScheme::TTypeId> ExtractTypes(const TVector<std::pair<TString, NScheme: return types; } -TString FromCells(const TConstArrayRef<TCell>& cells, const TVector<std::pair<TString, NScheme::TTypeId>>& columns) { +TString FromCells(const TConstArrayRef<TCell>& cells, const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns) { Y_VERIFY(cells.size() == columns.size()); if (cells.empty()) { return {}; } - TVector<NScheme::TTypeId> types = ExtractTypes(columns); + TVector<NScheme::TTypeInfo> types = ExtractTypes(columns); NArrow::TArrowBatchBuilder batchBuilder; batchBuilder.Reserve(1); @@ -338,9 +338,9 @@ using EOperation = NArrow::EOperation; using TPredicate = NOlap::TPredicate; std::pair<TPredicate, TPredicate> RangePredicates(const TSerializedTableRange& range, - const TVector<std::pair<TString, NScheme::TTypeId>>& columns) { + const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns) { TVector<TCell> leftCells; - TVector<std::pair<TString, NScheme::TTypeId>> leftColumns; + TVector<std::pair<TString, NScheme::TTypeInfo>> leftColumns; bool leftTrailingNull = false; { TConstArrayRef<TCell> cells = range.From.GetCells(); @@ -360,7 +360,7 @@ std::pair<TPredicate, TPredicate> RangePredicates(const TSerializedTableRange& r } TVector<TCell> rightCells; - TVector<std::pair<TString, NScheme::TTypeId>> rightColumns; + TVector<std::pair<TString, NScheme::TTypeInfo>> rightColumns; bool rightTrailingNull = false; { TConstArrayRef<TCell> cells = range.To.GetCells(); diff --git a/ydb/core/tx/columnshard/columnshard_common.h b/ydb/core/tx/columnshard/columnshard_common.h index fb515945dc5..6eabb89910e 100644 --- a/ydb/core/tx/columnshard/columnshard_common.h +++ b/ydb/core/tx/columnshard/columnshard_common.h @@ -15,7 +15,7 @@ namespace NKikimr::NColumnShard { using NOlap::TWriteId; std::pair<NOlap::TPredicate, NOlap::TPredicate> -RangePredicates(const TSerializedTableRange& range, const TVector<std::pair<TString, NScheme::TTypeId>>& columns); +RangePredicates(const TSerializedTableRange& range, const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns); class IColumnResolver { public: diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index efb22ed74e7..b0cb8d70e1a 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -1,5 +1,6 @@ #include "columnshard_impl.h" #include "columnshard_schema.h" +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/tx/columnshard/engines/column_engine_logs.h> #include <ydb/core/tablet/tablet_counters_protobuf.h> @@ -809,8 +810,9 @@ NOlap::TIndexInfo TColumnShard::ConvertSchema(const NKikimrSchemeOp::TColumnTabl for (const auto& col : schema.GetColumns()) { const ui32 id = col.GetId(); const TString& name = col.GetName(); - - indexInfo.Columns[id] = NTable::TColumn(name, id, col.GetTypeId()); + auto typeInfo = NScheme::TypeInfoFromProtoColumnType(col.GetTypeId(), + col.HasTypeInfo() ? &col.GetTypeInfo() : nullptr); + indexInfo.Columns[id] = NTable::TColumn(name, id, typeInfo); indexInfo.ColumnNames[name] = id; } diff --git a/ydb/core/tx/columnshard/columnshard_ut_common.cpp b/ydb/core/tx/columnshard/columnshard_ut_common.cpp index ef849697feb..786359eb054 100644 --- a/ydb/core/tx/columnshard/columnshard_ut_common.cpp +++ b/ydb/core/tx/columnshard/columnshard_ut_common.cpp @@ -4,6 +4,7 @@ #include <ydb/core/base/tablet.h> #include <ydb/core/base/tablet_resolver.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <library/cpp/testing/unittest/registar.h> namespace NKikimr::NTxUT { @@ -101,7 +102,13 @@ void ScanIndexStats(TTestBasicRuntime& runtime, TActorId& sender, const TVector< auto ydbSchema = PrimaryIndexStatsSchema; for (const auto& col : ydbSchema.Columns) { record.AddColumnTags(col.second.Id); - record.AddColumnTypes(col.second.PType); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(col.second.PType); + record.AddColumnTypes(columnType.TypeId); + if (columnType.TypeInfo) { + *record.AddColumnTypeInfos() = *columnType.TypeInfo; + } else { + *record.AddColumnTypeInfos() = NKikimrProto::TTypeInfo(); + } } for (ui64 pathId : pathIds) { @@ -155,11 +162,12 @@ void PlanCommit(TTestBasicRuntime& runtime, TActorId& sender, ui64 planStep, con } } -TVector<TCell> MakeTestCells(const TVector<TTypeId>& types, ui32 value, TVector<TString>& mem) { +TVector<TCell> MakeTestCells(const TVector<TTypeInfo>& types, ui32 value, TVector<TString>& mem) { TVector<TCell> cells; cells.reserve(types.size()); - for (auto& type : types) { + for (auto& typeInfo : types) { + auto type = typeInfo.GetTypeId(); if (type == NTypeIds::Utf8 || type == NTypeIds::String || type == NTypeIds::String4k || @@ -196,7 +204,7 @@ TVector<TCell> MakeTestCells(const TVector<TTypeId>& types, ui32 value, TVector< return cells; } -TString MakeTestBlob(std::pair<ui64, ui64> range, const TVector<std::pair<TString, TTypeId>>& columns, +TString MakeTestBlob(std::pair<ui64, ui64> range, const TVector<std::pair<TString, TTypeInfo>>& columns, const THashSet<TString>& nullColumns) { TString err; NArrow::TArrowBatchBuilder batchBuilder(arrow::Compression::LZ4_FRAME); @@ -210,7 +218,7 @@ TString MakeTestBlob(std::pair<ui64, ui64> range, const TVector<std::pair<TStrin } TVector<TString> mem; - TVector<TTypeId> types = TTestSchema::ExtractTypes(columns); + TVector<TTypeInfo> types = TTestSchema::ExtractTypes(columns); // insert, not ordered for (size_t i = range.first; i < range.second; i += 2) { TVector<TCell> cells = MakeTestCells(types, i, mem); @@ -240,9 +248,9 @@ TString MakeTestBlob(std::pair<ui64, ui64> range, const TVector<std::pair<TStrin } TSerializedTableRange MakeTestRange(std::pair<ui64, ui64> range, bool inclusiveFrom, bool inclusiveTo, - const TVector<std::pair<TString, TTypeId>>& columns) { + const TVector<std::pair<TString, TTypeInfo>>& columns) { TVector<TString> mem; - TVector<TTypeId> types = TTestSchema::ExtractTypes(columns); + TVector<TTypeInfo> types = TTestSchema::ExtractTypes(columns); TVector<TCell> cellsFrom = MakeTestCells(types, range.first, mem); TVector<TCell> cellsTo = MakeTestCells(types, range.second, mem); diff --git a/ydb/core/tx/columnshard/columnshard_ut_common.h b/ydb/core/tx/columnshard/columnshard_ut_common.h index 2fdb5c12a99..5cb92a8ae83 100644 --- a/ydb/core/tx/columnshard/columnshard_ut_common.h +++ b/ydb/core/tx/columnshard/columnshard_ut_common.h @@ -5,6 +5,7 @@ #include <ydb/core/formats/arrow_batch_builder.h> #include <ydb/core/scheme/scheme_tabledefs.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/testlib/tablet_helpers.h> #include <ydb/core/testlib/test_client.h> #include <library/cpp/testing/unittest/registar.h> @@ -21,6 +22,7 @@ public: namespace NTypeIds = NScheme::NTypeIds; using TTypeId = NScheme::TTypeId; +using TTypeInfo = NScheme::TTypeInfo; struct TTestSchema { static const constexpr char * DefaultTtlColumn = "saved_at"; @@ -87,93 +89,97 @@ struct TTestSchema { }; static auto YdbSchema() { - TVector<std::pair<TString, TTypeId>> schema = { + TVector<std::pair<TString, TTypeInfo>> schema = { // PK - {"timestamp", NTypeIds::Timestamp }, - {"resource_type", NTypeIds::Utf8 }, - {"resource_id", NTypeIds::Utf8 }, - {"uid", NTypeIds::Utf8 }, + {"timestamp", TTypeInfo(NTypeIds::Timestamp) }, + {"resource_type", TTypeInfo(NTypeIds::Utf8) }, + {"resource_id", TTypeInfo(NTypeIds::Utf8) }, + {"uid", TTypeInfo(NTypeIds::Utf8) }, // - {"level", NTypeIds::Int32 }, - {"message", NTypeIds::Utf8 }, - {"json_payload", NTypeIds::Json }, - {"ingested_at", NTypeIds::Timestamp }, - {"saved_at", NTypeIds::Timestamp }, - {"request_id", NTypeIds::Utf8 } + {"level", TTypeInfo(NTypeIds::Int32) }, + {"message", TTypeInfo(NTypeIds::Utf8) }, + {"json_payload", TTypeInfo(NTypeIds::Json) }, + {"ingested_at", TTypeInfo(NTypeIds::Timestamp) }, + {"saved_at", TTypeInfo(NTypeIds::Timestamp) }, + {"request_id", TTypeInfo(NTypeIds::Utf8) } }; return schema; }; static auto YdbExoticSchema() { - TVector<std::pair<TString, TTypeId>> schema = { + TVector<std::pair<TString, TTypeInfo>> schema = { // PK - {"timestamp", NTypeIds::Timestamp }, - {"resource_type", NTypeIds::Utf8 }, - {"resource_id", NTypeIds::Utf8 }, - {"uid", NTypeIds::Utf8 }, + {"timestamp", TTypeInfo(NTypeIds::Timestamp) }, + {"resource_type", TTypeInfo(NTypeIds::Utf8) }, + {"resource_id", TTypeInfo(NTypeIds::Utf8) }, + {"uid", TTypeInfo(NTypeIds::Utf8) }, // - {"level", NTypeIds::Int32 }, - {"message", NTypeIds::String4k }, - {"json_payload", NTypeIds::JsonDocument }, - {"ingested_at", NTypeIds::Timestamp }, - {"saved_at", NTypeIds::Timestamp }, - {"request_id", NTypeIds::Yson } + {"level", TTypeInfo(NTypeIds::Int32) }, + {"message", TTypeInfo(NTypeIds::String4k) }, + {"json_payload", TTypeInfo(NTypeIds::JsonDocument) }, + {"ingested_at", TTypeInfo(NTypeIds::Timestamp) }, + {"saved_at", TTypeInfo(NTypeIds::Timestamp) }, + {"request_id", TTypeInfo(NTypeIds::Yson) } }; return schema; }; static auto YdbPkSchema() { - TVector<std::pair<TString, TTypeId>> schema = { - {"timestamp", NTypeIds::Timestamp }, - {"resource_type", NTypeIds::Utf8 }, - {"resource_id", NTypeIds::Utf8 }, - {"uid", NTypeIds::Utf8 } + TVector<std::pair<TString, TTypeInfo>> schema = { + {"timestamp", TTypeInfo(NTypeIds::Timestamp) }, + {"resource_type", TTypeInfo(NTypeIds::Utf8) }, + {"resource_id", TTypeInfo(NTypeIds::Utf8) }, + {"uid", TTypeInfo(NTypeIds::Utf8) } }; return schema; } static auto YdbAllTypesSchema() { - TVector<std::pair<TString, TTypeId>> schema = { - { "ts", NTypeIds::Timestamp }, - - { "i8", NTypeIds::Int8 }, - { "i16", NTypeIds::Int16 }, - { "i32", NTypeIds::Int32 }, - { "i64", NTypeIds::Int64 }, - { "u8", NTypeIds::Uint8 }, - { "u16", NTypeIds::Uint16 }, - { "u32", NTypeIds::Uint32 }, - { "u64", NTypeIds::Uint64 }, - { "float", NTypeIds::Float }, - { "double", NTypeIds::Double }, - - { "byte", NTypeIds::Byte }, - //{ "bool", NTypeIds::Bool }, - //{ "decimal", NTypeIds::Decimal }, - //{ "dynum", NTypeIds::DyNumber }, - - { "date", NTypeIds::Date }, - { "datetime", NTypeIds::Datetime }, - //{ "interval", NTypeIds::Interval }, - - {"text", NTypeIds::Text }, - {"bytes", NTypeIds::Bytes }, - {"yson", NTypeIds::Yson }, - {"json", NTypeIds::Json }, - {"jsondoc", NTypeIds::JsonDocument } + TVector<std::pair<TString, TTypeInfo>> schema = { + { "ts", TTypeInfo(NTypeIds::Timestamp) }, + + { "i8", TTypeInfo(NTypeIds::Int8) }, + { "i16", TTypeInfo(NTypeIds::Int16) }, + { "i32", TTypeInfo(NTypeIds::Int32) }, + { "i64", TTypeInfo(NTypeIds::Int64) }, + { "u8", TTypeInfo(NTypeIds::Uint8) }, + { "u16", TTypeInfo(NTypeIds::Uint16) }, + { "u32", TTypeInfo(NTypeIds::Uint32) }, + { "u64", TTypeInfo(NTypeIds::Uint64) }, + { "float", TTypeInfo(NTypeIds::Float) }, + { "double", TTypeInfo(NTypeIds::Double) }, + + { "byte", TTypeInfo(NTypeIds::Byte) }, + //{ "bool", TTypeInfo(NTypeIds::Bool) }, + //{ "decimal", TTypeInfo(NTypeIds::Decimal) }, + //{ "dynum", TTypeInfo(NTypeIds::DyNumber) }, + + { "date", TTypeInfo(NTypeIds::Date) }, + { "datetime", TTypeInfo(NTypeIds::Datetime) }, + //{ "interval", TTypeInfo(NTypeIds::Interval) }, + + {"text", TTypeInfo(NTypeIds::Text) }, + {"bytes", TTypeInfo(NTypeIds::Bytes) }, + {"yson", TTypeInfo(NTypeIds::Yson) }, + {"json", TTypeInfo(NTypeIds::Json) }, + {"jsondoc", TTypeInfo(NTypeIds::JsonDocument) } }; return schema; }; - static NKikimrSchemeOp::TOlapColumnDescription CreateColumn(ui32 id, const TString& name, TTypeId type) { + static NKikimrSchemeOp::TOlapColumnDescription CreateColumn(ui32 id, const TString& name, TTypeInfo type) { NKikimrSchemeOp::TOlapColumnDescription col; col.SetId(id); col.SetName(name); - col.SetTypeId(type); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(type); + col.SetTypeId(columnType.TypeId); + if (columnType.TypeInfo) { + *col.MutableTypeInfo() = *columnType.TypeInfo; + } return col; } - static TString CreateTableTxBody(ui64 pathId, const TVector<std::pair<TString, TTypeId>>& columns, + static TString CreateTableTxBody(ui64 pathId, const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns, const TTableSpecials& specials = {}) { NKikimrTxColumnShard::TSchemaTxBody tx; auto* table = tx.MutableEnsureTables()->AddTables(); @@ -310,7 +316,7 @@ struct TTestSchema { return txBody; } - static TVector<TString> ExtractNames(const TVector<std::pair<TString, TTypeId>>& columns) { + static TVector<TString> ExtractNames(const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns) { TVector<TString> out; out.reserve(columns.size()); for (auto& col : columns) { @@ -319,8 +325,8 @@ struct TTestSchema { return out; } - static TVector<TTypeId> ExtractTypes(const TVector<std::pair<TString, TTypeId>>& columns) { - TVector<TTypeId> types; + static TVector<NScheme::TTypeInfo> ExtractTypes(const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns) { + TVector<NScheme::TTypeInfo> types; types.reserve(columns.size()); for (auto& [name, type] : columns) { types.push_back(type); @@ -344,9 +350,9 @@ inline void PlanCommit(TTestBasicRuntime& runtime, TActorId& sender, ui64 planSt PlanCommit(runtime, sender, planStep, ids); } -TString MakeTestBlob(std::pair<ui64, ui64> range, const TVector<std::pair<TString, TTypeId>>& columns, +TString MakeTestBlob(std::pair<ui64, ui64> range, const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns, const THashSet<TString>& nullColumns = {}); TSerializedTableRange MakeTestRange(std::pair<ui64, ui64> range, bool inclusiveFrom, bool inclusiveTo, - const TVector<std::pair<TString, TTypeId>>& columns); + const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns); } diff --git a/ydb/core/tx/columnshard/engines/index_info.cpp b/ydb/core/tx/columnshard/engines/index_info.cpp index f13157b96cb..e7400f255ab 100644 --- a/ydb/core/tx/columnshard/engines/index_info.cpp +++ b/ydb/core/tx/columnshard/engines/index_info.cpp @@ -97,6 +97,7 @@ TVector<TRawTypeValue> TIndexInfo::ExtractKey(const THashMap<ui32, TCell>& field const TCell& cell = it->second; Y_VERIFY(allowNulls || !cell.IsNull()); + key.emplace_back(TRawTypeValue(cell.AsRef(), column.PType)); } diff --git a/ydb/core/tx/columnshard/engines/index_info.h b/ydb/core/tx/columnshard/engines/index_info.h index ffb108c68b9..7484a05c9a9 100644 --- a/ydb/core/tx/columnshard/engines/index_info.h +++ b/ydb/core/tx/columnshard/engines/index_info.h @@ -40,9 +40,9 @@ static std::shared_ptr<arrow::Schema> MakeArrowSchema(const NTable::TScheme::TTa } inline -TVector<std::pair<TString, NScheme::TTypeId>> +TVector<std::pair<TString, NScheme::TTypeInfo>> GetColumns(const NTable::TScheme::TTableSchema& tableSchema, const TVector<ui32>& ids) { - TVector<std::pair<TString, NScheme::TTypeId>> out; + TVector<std::pair<TString, NScheme::TTypeInfo>> out; out.reserve(ids.size()); for (ui32 id : ids) { Y_VERIFY(tableSchema.Columns.count(id)); @@ -127,17 +127,17 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema { return out; } - TVector<std::pair<TString, NScheme::TTypeId>> GetColumns(const TVector<ui32>& ids) const { + TVector<std::pair<TString, NScheme::TTypeInfo>> GetColumns(const TVector<ui32>& ids) const { return NOlap::GetColumns(*this, ids); } // Traditional Primary Key (includes uniqueness, search and sorting logic) - TVector<std::pair<TString, NScheme::TTypeId>> GetPK() const { + TVector<std::pair<TString, NScheme::TTypeInfo>> GetPK() const { return GetColumns(KeyColumns); } - static TVector<std::pair<TString, NScheme::TTypeId>> SchemaIndexStats(ui32 version = 0); - static TVector<std::pair<TString, NScheme::TTypeId>> SchemaIndexStatsKey(ui32 version = 0); + static TVector<std::pair<TString, NScheme::TTypeInfo>> SchemaIndexStats(ui32 version = 0); + static TVector<std::pair<TString, NScheme::TTypeInfo>> SchemaIndexStatsKey(ui32 version = 0); ui32 GetPKFirstColumnId() const { Y_VERIFY(KeyColumns.size()); @@ -153,7 +153,7 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema { const std::shared_ptr<arrow::Schema>& GetIndexKey() const { return IndexKey; } void SetAllKeys(const TVector<TString>& columns, const TVector<int>& indexKeyPos); - void SetAllKeys(const TVector<std::pair<TString, NScheme::TTypeId>>& columns, const TVector<int>& indexKeyPos) { + void SetAllKeys(const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns, const TVector<int>& indexKeyPos) { SetAllKeys(NamesOnly(columns), indexKeyPos); } @@ -261,7 +261,7 @@ private: } } - static TVector<TString> NamesOnly(const TVector<std::pair<TString, NScheme::TTypeId>>& columns) { + static TVector<TString> NamesOnly(const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns) { TVector<TString> out; out.reserve(columns.size()); for (auto& [name, type] : columns) { diff --git a/ydb/core/tx/columnshard/engines/indexed_read_data.cpp b/ydb/core/tx/columnshard/engines/indexed_read_data.cpp index bbc8823f095..4b98d5329cc 100644 --- a/ydb/core/tx/columnshard/engines/indexed_read_data.cpp +++ b/ydb/core/tx/columnshard/engines/indexed_read_data.cpp @@ -128,11 +128,11 @@ std::unique_ptr<NColumnShard::TScanIteratorBase> TReadMetadata::StartScan() cons } -TVector<std::pair<TString, NScheme::TTypeId>> TReadStatsMetadata::GetResultYqlSchema() const { +TVector<std::pair<TString, NScheme::TTypeInfo>> TReadStatsMetadata::GetResultYqlSchema() const { return NOlap::GetColumns(NColumnShard::PrimaryIndexStatsSchema, ResultColumnIds); } -TVector<std::pair<TString, NScheme::TTypeId>> TReadStatsMetadata::GetKeyYqlSchema() const { +TVector<std::pair<TString, NScheme::TTypeInfo>> TReadStatsMetadata::GetKeyYqlSchema() const { return NOlap::GetColumns(NColumnShard::PrimaryIndexStatsSchema, NColumnShard::PrimaryIndexStatsSchema.KeyColumns); } diff --git a/ydb/core/tx/columnshard/engines/indexed_read_data.h b/ydb/core/tx/columnshard/engines/indexed_read_data.h index 0c7818bbbab..ecb594caa11 100644 --- a/ydb/core/tx/columnshard/engines/indexed_read_data.h +++ b/ydb/core/tx/columnshard/engines/indexed_read_data.h @@ -56,8 +56,8 @@ struct TReadMetadataBase { bool IsSorted() const { return IsAscSorted() || IsDescSorted(); } void SetDescSorting() { Sorting = ESorting::DESC; } - virtual TVector<std::pair<TString, NScheme::TTypeId>> GetResultYqlSchema() const = 0; - virtual TVector<std::pair<TString, NScheme::TTypeId>> GetKeyYqlSchema() const = 0; + virtual TVector<std::pair<TString, NScheme::TTypeInfo>> GetResultYqlSchema() const = 0; + virtual TVector<std::pair<TString, NScheme::TTypeInfo>> GetKeyYqlSchema() const = 0; virtual std::unique_ptr<NColumnShard::TScanIteratorBase> StartScan() const = 0; virtual void Dump(IOutputStream& out) const { Y_UNUSED(out); }; @@ -102,7 +102,7 @@ struct TReadMetadata : public TReadMetadataBase, public std::enable_shared_from_ return IndexInfo.GetReplaceKey(); } - TVector<std::pair<TString, NScheme::TTypeId>> GetResultYqlSchema() const override { + TVector<std::pair<TString, NScheme::TTypeInfo>> GetResultYqlSchema() const override { TVector<NTable::TTag> columnIds; columnIds.reserve(ResultSchema->num_fields()); for (const auto& field: ResultSchema->fields()) { @@ -112,7 +112,7 @@ struct TReadMetadata : public TReadMetadataBase, public std::enable_shared_from_ return IndexInfo.GetColumns(columnIds); } - TVector<std::pair<TString, NScheme::TTypeId>> GetKeyYqlSchema() const override { + TVector<std::pair<TString, NScheme::TTypeInfo>> GetKeyYqlSchema() const override { return IndexInfo.GetPK(); } @@ -165,9 +165,9 @@ struct TReadStatsMetadata : public TReadMetadataBase, public std::enable_shared_ : TabletId(tabletId) {} - TVector<std::pair<TString, NScheme::TTypeId>> GetResultYqlSchema() const override; + TVector<std::pair<TString, NScheme::TTypeInfo>> GetResultYqlSchema() const override; - TVector<std::pair<TString, NScheme::TTypeId>> GetKeyYqlSchema() const override; + TVector<std::pair<TString, NScheme::TTypeInfo>> GetKeyYqlSchema() const override; std::unique_ptr<NColumnShard::TScanIteratorBase> StartScan() const override; }; diff --git a/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp b/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp index fb3546293ce..953e700a5d6 100644 --- a/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp +++ b/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp @@ -8,6 +8,7 @@ namespace NKikimr { using namespace NOlap; namespace NTypeIds = NScheme::NTypeIds; using TTypeId = NScheme::TTypeId; +using TTypeInfo = NScheme::TTypeInfo; namespace { @@ -153,21 +154,21 @@ private: THashMap<ui32, TIndex> Indices; }; -static const TVector<std::pair<TString, TTypeId>> testColumns = { +static const TVector<std::pair<TString, TTypeInfo>> testColumns = { // PK - {"timestamp", NTypeIds::Timestamp }, - {"resource_type", NTypeIds::Utf8 }, - {"resource_id", NTypeIds::Utf8 }, - {"uid", NTypeIds::Utf8 }, + {"timestamp", TTypeInfo(NTypeIds::Timestamp) }, + {"resource_type", TTypeInfo(NTypeIds::Utf8) }, + {"resource_id", TTypeInfo(NTypeIds::Utf8) }, + {"uid", TTypeInfo(NTypeIds::Utf8) }, // - {"message", NTypeIds::Utf8 } + {"message", TTypeInfo(NTypeIds::Utf8) } }; -static const TVector<std::pair<TString, TTypeId>> testKey = { - {"timestamp", NTypeIds::Timestamp }, - {"resource_type", NTypeIds::Utf8 }, - {"resource_id", NTypeIds::Utf8 }, - {"uid", NTypeIds::Utf8 } +static const TVector<std::pair<TString, TTypeInfo>> testKey = { + {"timestamp", TTypeInfo(NTypeIds::Timestamp) }, + {"resource_type", TTypeInfo(NTypeIds::Utf8) }, + {"resource_id", TTypeInfo(NTypeIds::Utf8) }, + {"uid", TTypeInfo(NTypeIds::Utf8) } }; TIndexInfo TestTableInfo() { diff --git a/ydb/core/tx/columnshard/ut/CMakeLists.darwin.txt b/ydb/core/tx/columnshard/ut/CMakeLists.darwin.txt index 64503d0cb59..e61780de694 100644 --- a/ydb/core/tx/columnshard/ut/CMakeLists.darwin.txt +++ b/ydb/core/tx/columnshard/ut/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-columnshard-ut PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx public-lib-yson_value ) diff --git a/ydb/core/tx/columnshard/ut/CMakeLists.linux.txt b/ydb/core/tx/columnshard/ut/CMakeLists.linux.txt index 254a9058ecc..62ba940e34b 100644 --- a/ydb/core/tx/columnshard/ut/CMakeLists.linux.txt +++ b/ydb/core/tx/columnshard/ut/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-columnshard-ut PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx public-lib-yson_value ) diff --git a/ydb/core/tx/columnshard/ut_columnshard_read_write.cpp b/ydb/core/tx/columnshard/ut_columnshard_read_write.cpp index 2a28f40711a..7dda4cdcdb8 100644 --- a/ydb/core/tx/columnshard/ut_columnshard_read_write.cpp +++ b/ydb/core/tx/columnshard/ut_columnshard_read_write.cpp @@ -16,9 +16,10 @@ namespace namespace NTypeIds = NScheme::NTypeIds; using TTypeId = NScheme::TTypeId; +using TTypeInfo = NScheme::TTypeInfo; -static const TVector<std::pair<TString, TTypeId>> testYdbSchema = TTestSchema::YdbSchema(); -static const TVector<std::pair<TString, TTypeId>> testYdbPkSchema = TTestSchema::YdbPkSchema(); +static const TVector<std::pair<TString, TTypeInfo>> testYdbSchema = TTestSchema::YdbSchema(); +static const TVector<std::pair<TString, TTypeInfo>> testYdbPkSchema = TTestSchema::YdbPkSchema(); bool DataHas(const TVector<TString>& blobs, const TString& srtSchema, std::pair<ui64, ui64> range, bool requireUniq = false) { @@ -210,7 +211,7 @@ bool CheckColumns(const TString& blob, const NKikimrTxColumnShard::TMetadata& me } void SetupSchema(TTestBasicRuntime& runtime, TActorId& sender, ui64 pathId, - const TVector<std::pair<TString, TTypeId>>& schema = TTestSchema::YdbSchema(), + const TVector<std::pair<TString, TTypeInfo>>& schema = TTestSchema::YdbSchema(), NOlap::TSnapshot snap = {10, 10}, TString codec = "") { bool ok = ProposeSchemaTx(runtime, sender, TTestSchema::CreateTableTxBody(pathId, schema, @@ -220,7 +221,7 @@ void SetupSchema(TTestBasicRuntime& runtime, TActorId& sender, ui64 pathId, PlanSchemaTx(runtime, sender, snap); } -void TestWrite(const TVector<std::pair<TString, TTypeId>>& ydbSchema) { +void TestWrite(const TVector<std::pair<TString, TTypeInfo>>& ydbSchema) { TTestBasicRuntime runtime; TTester::Setup(runtime); @@ -241,7 +242,7 @@ void TestWrite(const TVector<std::pair<TString, TTypeId>>& ydbSchema) { bool ok = WriteData(runtime, sender, metaShard, writeId, tableId, MakeTestBlob({0, 100}, ydbSchema)); UNIT_ASSERT(ok); - TVector<std::pair<TString, TTypeId>> schema = ydbSchema; + TVector<std::pair<TString, TTypeInfo>> schema = ydbSchema; // no data @@ -263,29 +264,29 @@ void TestWrite(const TVector<std::pair<TString, TTypeId>>& ydbSchema) { // TODO: better check (it probably does not work in general case) schema = ydbSchema; - schema[1].second = NTypeIds::Int32; + schema[1].second = TTypeInfo(NTypeIds::Int32); ok = WriteData(runtime, sender, metaShard, writeId, tableId, MakeTestBlob({0, 100}, schema)); UNIT_ASSERT(!ok); schema = ydbSchema; - schema[1].second = NTypeIds::Utf8; - schema[5].second = NTypeIds::Int32; + schema[1].second = TTypeInfo(NTypeIds::Utf8); + schema[5].second = TTypeInfo(NTypeIds::Int32); ok = WriteData(runtime, sender, metaShard, writeId, tableId, MakeTestBlob({0, 100}, schema)); UNIT_ASSERT(!ok); // reordered columns schema.resize(0); - schema.push_back({"level", NTypeIds::Int32 }); - schema.push_back({"timestamp", NTypeIds::Timestamp }); - schema.push_back({"uid", NTypeIds::Utf8 }); - schema.push_back({"resource_id", NTypeIds::Utf8 }); - schema.push_back({"resource_type", NTypeIds::Utf8 }); - schema.push_back({"message", NTypeIds::Utf8 }); - schema.push_back({"request_id", NTypeIds::Utf8 }); - schema.push_back({"saved_at", NTypeIds::Timestamp }); - schema.push_back({"ingested_at", NTypeIds::Timestamp }); - schema.push_back({"json_payload", NTypeIds::Json }); + schema.push_back({"level", TTypeInfo(NTypeIds::Int32) }); + schema.push_back({"timestamp", TTypeInfo(NTypeIds::Timestamp) }); + schema.push_back({"uid", TTypeInfo(NTypeIds::Utf8) }); + schema.push_back({"resource_id", TTypeInfo(NTypeIds::Utf8) }); + schema.push_back({"resource_type", TTypeInfo(NTypeIds::Utf8) }); + schema.push_back({"message", TTypeInfo(NTypeIds::Utf8) }); + schema.push_back({"request_id", TTypeInfo(NTypeIds::Utf8) }); + schema.push_back({"saved_at", TTypeInfo(NTypeIds::Timestamp) }); + schema.push_back({"ingested_at", TTypeInfo(NTypeIds::Timestamp) }); + schema.push_back({"json_payload", TTypeInfo(NTypeIds::Json) }); ok = WriteData(runtime, sender, metaShard, writeId, tableId, MakeTestBlob({0, 100}, schema)); UNIT_ASSERT(!ok); @@ -363,7 +364,7 @@ void TestWriteReadDup() { } } -void TestWriteRead(bool reboots, const TVector<std::pair<TString, TTypeId>>& ydbSchema = TTestSchema::YdbSchema(), +void TestWriteRead(bool reboots, const TVector<std::pair<TString, TTypeInfo>>& ydbSchema = TTestSchema::YdbSchema(), TString codec = "") { TTestBasicRuntime runtime; TTester::Setup(runtime); @@ -1084,7 +1085,7 @@ NKikimrSSA::TProgram MakeSelectAggregatesWithFilter(ui32 columnId, ui32 filterCo return ssa; } -void TestReadWithProgram(const TVector<std::pair<TString, TTypeId>>& ydbSchema = TTestSchema::YdbSchema()) +void TestReadWithProgram(const TVector<std::pair<TString, TTypeInfo>>& ydbSchema = TTestSchema::YdbSchema()) { TTestBasicRuntime runtime; TTester::Setup(runtime); @@ -1219,7 +1220,7 @@ struct TReadAggregateResult { std::vector<int64_t> Counts = {100}; }; -void TestReadAggregate(const TVector<std::pair<TString, TTypeId>>& ydbSchema, const TString& testDataBlob, +void TestReadAggregate(const TVector<std::pair<TString, TTypeInfo>>& ydbSchema, const TString& testDataBlob, bool addProjection, const std::vector<ui32>& aggKeys = {}, const TReadAggregateResult& expectedResult = {}, const TReadAggregateResult& expectedFiltered = {1, {1}, {1}, {1}}) { @@ -1266,8 +1267,8 @@ void TestReadAggregate(const TVector<std::pair<TString, TTypeId>>& ydbSchema, co ui32 prog = 0; for (ui32 i = 0; i < ydbSchema.size(); ++i, ++prog) { - if (intTypes.count(ydbSchema[i].second) || - strTypes.count(ydbSchema[i].second)) { + if (intTypes.count(ydbSchema[i].second.GetTypeId()) || + strTypes.count(ydbSchema[i].second.GetTypeId())) { checkResult.insert(prog); } @@ -1283,8 +1284,8 @@ void TestReadAggregate(const TVector<std::pair<TString, TTypeId>>& ydbSchema, co for (ui32 i = 0; i < ydbSchema.size(); ++i, ++prog) { isFiltered.insert(prog); - if (intTypes.count(ydbSchema[i].second) || - strTypes.count(ydbSchema[i].second)) { + if (intTypes.count(ydbSchema[i].second.GetTypeId()) || + strTypes.count(ydbSchema[i].second.GetTypeId())) { checkResult.insert(prog); } @@ -1445,7 +1446,7 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { Cerr << "-- group by key: " << key << "\n"; // the type has the same values in test batch so result would be grouped in one row - if (sameValTypes.count(schema[key].second)) { + if (sameValTypes.count(schema[key].second.GetTypeId())) { TestReadAggregate(schema, testBlob, (key % 2), {key}, resGrouped, resFiltered); } else { TestReadAggregate(schema, testBlob, (key % 2), {key}, resDefault, resFiltered); @@ -1453,8 +1454,8 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { } for (ui32 key = 0; key < schema.size() - 1; ++key) { Cerr << "-- group by key: " << key << ", " << key + 1 << "\n"; - if (sameValTypes.count(schema[key].second) && - sameValTypes.count(schema[key + 1].second)) { + if (sameValTypes.count(schema[key].second.GetTypeId()) && + sameValTypes.count(schema[key + 1].second.GetTypeId())) { TestReadAggregate(schema, testBlob, (key % 2), {key, key + 1}, resGrouped, resFiltered); } else { TestReadAggregate(schema, testBlob, (key % 2), {key, key + 1}, resDefault, resFiltered); @@ -1462,9 +1463,9 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { } for (ui32 key = 0; key < schema.size() - 2; ++key) { Cerr << "-- group by key: " << key << ", " << key + 1 << ", " << key + 2 << "\n"; - if (sameValTypes.count(schema[key].second) && - sameValTypes.count(schema[key + 1].second) && - sameValTypes.count(schema[key + 1].second)) { + if (sameValTypes.count(schema[key].second.GetTypeId()) && + sameValTypes.count(schema[key + 1].second.GetTypeId()) && + sameValTypes.count(schema[key + 1].second.GetTypeId())) { TestReadAggregate(schema, testBlob, (key % 2), {key, key + 1, key + 2}, resGrouped, resFiltered); } else { TestReadAggregate(schema, testBlob, (key % 2), {key, key + 1, key + 2}, resDefault, resFiltered); diff --git a/ydb/core/tx/columnshard/ut_columnshard_schema.cpp b/ydb/core/tx/columnshard/ut_columnshard_schema.cpp index 18ca9b217df..ecf54d53e54 100644 --- a/ydb/core/tx/columnshard/ut_columnshard_schema.cpp +++ b/ydb/core/tx/columnshard/ut_columnshard_schema.cpp @@ -11,7 +11,7 @@ using NWrappers::NTestHelpers::TS3Mock; namespace { -static const TVector<std::pair<TString, TTypeId>> testYdbSchema = TTestSchema::YdbSchema(); +static const TVector<std::pair<TString, TTypeInfo>> testYdbSchema = TTestSchema::YdbSchema(); std::shared_ptr<arrow::RecordBatch> UpdateColumn(std::shared_ptr<arrow::RecordBatch> batch, TString columnName, i64 seconds) { std::string name(columnName.c_str(), columnName.size()); diff --git a/ydb/core/tx/coordinator/ut/CMakeLists.darwin.txt b/ydb/core/tx/coordinator/ut/CMakeLists.darwin.txt index 56e660d083a..f536fdf3527 100644 --- a/ydb/core/tx/coordinator/ut/CMakeLists.darwin.txt +++ b/ydb/core/tx/coordinator/ut/CMakeLists.darwin.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-core-tx-coordinator-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main core-tx-coordinator - ydb-core-testlib + core-testlib-default ydb-core-tx ) target_link_options(ydb-core-tx-coordinator-ut PRIVATE diff --git a/ydb/core/tx/coordinator/ut/CMakeLists.linux.txt b/ydb/core/tx/coordinator/ut/CMakeLists.linux.txt index f1e03b81d94..5c6ce8891b1 100644 --- a/ydb/core/tx/coordinator/ut/CMakeLists.linux.txt +++ b/ydb/core/tx/coordinator/ut/CMakeLists.linux.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-tx-coordinator-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main core-tx-coordinator - ydb-core-testlib + core-testlib-default ydb-core-tx ) target_link_options(ydb-core-tx-coordinator-ut PRIVATE diff --git a/ydb/core/tx/datashard/build_distributed_erase_tx_out_rs_unit.cpp b/ydb/core/tx/datashard/build_distributed_erase_tx_out_rs_unit.cpp index ee90c2fc2b8..aea50ffbe3c 100644 --- a/ydb/core/tx/datashard/build_distributed_erase_tx_out_rs_unit.cpp +++ b/ydb/core/tx/datashard/build_distributed_erase_tx_out_rs_unit.cpp @@ -61,8 +61,7 @@ class TBuildDistributedEraseTxOutRSUnit : public TExecutionUnit { for (ui32 pos = 0; pos < expectedValue.size(); ++pos) { const auto& expected = expectedValue.at(pos); const auto& actual = actualValue.at(pos); - - if (0 != CompareTypedCells(actual, expected.AsRef(), expected.Type())) { + if (0 != CompareTypedCells(actual, expected.AsRef(), expected.TypeInfo())) { return false; } } @@ -113,9 +112,8 @@ public: TVector<TRawTypeValue> key; for (ui32 pos = 0; pos < tableInfo.KeyColumnTypes.size(); ++pos) { - const NScheme::TTypeId type = tableInfo.KeyColumnTypes[pos]; + const NScheme::TTypeInfo type = tableInfo.KeyColumnTypes[pos]; const TCell& cell = keyCells.GetCells()[pos]; - key.emplace_back(TRawTypeValue(cell.AsRef(), type)); } @@ -130,7 +128,7 @@ public: auto it = tableInfo.Columns.find(eraseTx->GetIndexColumnIds().Get(pos)); Y_VERIFY(it != tableInfo.Columns.end()); - const NScheme::TTypeId type = it->second.Type; + const NScheme::TTypeInfo type = it->second.Type; const TCell& cell = indexCells.GetCells()[pos]; indexTypedVals.emplace_back(TRawTypeValue(cell.AsRef(), type)); diff --git a/ydb/core/tx/datashard/change_collector_async_index.cpp b/ydb/core/tx/datashard/change_collector_async_index.cpp index d012f961899..d2f5400366b 100644 --- a/ydb/core/tx/datashard/change_collector_async_index.cpp +++ b/ydb/core/tx/datashard/change_collector_async_index.cpp @@ -237,7 +237,7 @@ TArrayRef<TTag> TAsyncIndexChangeCollector::GetTagsToSelect(const TTableId& tabl } } -void TAsyncIndexChangeCollector::FillKeyFromRowState(TTag tag, TPos pos, const TRowState& rowState, NScheme::TTypeId type) { +void TAsyncIndexChangeCollector::FillKeyFromRowState(TTag tag, TPos pos, const TRowState& rowState, NScheme::TTypeInfo type) { Y_VERIFY(pos < rowState.Size()); IndexKeyVals.emplace_back(rowState.Get(pos).AsRef(), type); @@ -267,7 +267,7 @@ void TAsyncIndexChangeCollector::FillKeyFromUpdate(TTag tag, TPos pos, TArrayRef TagsSeen.insert(tag); } -void TAsyncIndexChangeCollector::FillKeyWithNull(TTag tag, NScheme::TTypeId type) { +void TAsyncIndexChangeCollector::FillKeyWithNull(TTag tag, NScheme::TTypeInfo type) { IndexKeyVals.emplace_back(TRawTypeValue({}, type)); IndexKeyTags.emplace_back(tag); TagsSeen.insert(tag); @@ -282,7 +282,7 @@ void TAsyncIndexChangeCollector::FillDataFromUpdate(TTag tag, TPos pos, TArrayRe IndexDataVals.emplace_back(tag, ECellOp::Set, update.Value); } -void TAsyncIndexChangeCollector::FillDataWithNull(TTag tag, NScheme::TTypeId type) { +void TAsyncIndexChangeCollector::FillDataWithNull(TTag tag, NScheme::TTypeInfo type) { IndexDataVals.emplace_back(tag, ECellOp::Set, TRawTypeValue({}, type)); } diff --git a/ydb/core/tx/datashard/change_collector_async_index.h b/ydb/core/tx/datashard/change_collector_async_index.h index c6404f03f96..273e2316015 100644 --- a/ydb/core/tx/datashard/change_collector_async_index.h +++ b/ydb/core/tx/datashard/change_collector_async_index.h @@ -35,12 +35,12 @@ class TAsyncIndexChangeCollector: public TBaseChangeCollector { const THashMap<NTable::TTag, NTable::TPos>& GetKeyTagToPos(const TTableId& tableId) const; TArrayRef<NTable::TTag> GetTagsToSelect(const TTableId& tableId, NTable::ERowOp rop) const; - void FillKeyFromRowState(NTable::TTag tag, NTable::TPos pos, const NTable::TRowState& rowState, NScheme::TTypeId type); + void FillKeyFromRowState(NTable::TTag tag, NTable::TPos pos, const NTable::TRowState& rowState, NScheme::TTypeInfo type); void FillKeyFromKey(NTable::TTag tag, NTable::TPos pos, TArrayRef<const TRawTypeValue> key); void FillKeyFromUpdate(NTable::TTag tag, NTable::TPos pos, TArrayRef<const NTable::TUpdateOp> updates); - void FillKeyWithNull(NTable::TTag tag, NScheme::TTypeId type); + void FillKeyWithNull(NTable::TTag tag, NScheme::TTypeInfo type); void FillDataFromUpdate(NTable::TTag tag, NTable::TPos pos, TArrayRef<const NTable::TUpdateOp> updates); - void FillDataWithNull(NTable::TTag tag, NScheme::TTypeId type); + void FillDataWithNull(NTable::TTag tag, NScheme::TTypeInfo type); void Persist(const TTableId& tableId, const TPathId& pathId, NTable::ERowOp rop, TArrayRef<const TRawTypeValue> key, TArrayRef<const NTable::TTag> keyTags, diff --git a/ydb/core/tx/datashard/change_record.cpp b/ydb/core/tx/datashard/change_record.cpp index 1ec9cb145e1..622b6fcb4af 100644 --- a/ydb/core/tx/datashard/change_record.cpp +++ b/ydb/core/tx/datashard/change_record.cpp @@ -48,12 +48,12 @@ static NJson::TJsonValue YsonToJson(TStringBuf in) { return result; } -static NJson::TJsonValue ToJson(const TCell& cell, NScheme::TTypeId type) { +static NJson::TJsonValue ToJson(const TCell& cell, NScheme::TTypeInfo type) { if (cell.IsNull()) { return NJson::TJsonValue(NJson::JSON_NULL); } - switch (type) { + switch (type.GetTypeId()) { case NScheme::NTypeIds::Bool: return NJson::TJsonValue(cell.AsValue<bool>()); case NScheme::NTypeIds::Int8: @@ -100,6 +100,9 @@ static NJson::TJsonValue ToJson(const TCell& cell, NScheme::TTypeId type) { return StringToJson(NBinaryJson::SerializeToJson(cell.AsBuf())); case NScheme::NTypeIds::Yson: return YsonToJson(cell.AsBuf()); + case NScheme::NTypeIds::Pg: + // TODO: support pg types + Y_FAIL("pg types are not supported"); default: Y_FAIL("Unexpected type"); } diff --git a/ydb/core/tx/datashard/change_sender_async_index.cpp b/ydb/core/tx/datashard/change_sender_async_index.cpp index 12c357d7dc5..bc1d87abe85 100644 --- a/ydb/core/tx/datashard/change_sender_async_index.cpp +++ b/ydb/core/tx/datashard/change_sender_async_index.cpp @@ -513,7 +513,7 @@ class TAsyncIndexChangeSenderMain: public TActorBootstrapped<TAsyncIndexChangeSe } TagMap.clear(); - TVector<NYql::NUdf::TDataTypeId> keyColumnTypes; + TVector<NScheme::TTypeInfo> keyColumnTypes; for (const auto& [tag, column] : entry.Columns) { auto it = MainColumnToTag.find(column.Name); diff --git a/ydb/core/tx/datashard/change_sender_cdc_stream.cpp b/ydb/core/tx/datashard/change_sender_cdc_stream.cpp index 145778306e4..73fe7d13fc0 100644 --- a/ydb/core/tx/datashard/change_sender_cdc_stream.cpp +++ b/ydb/core/tx/datashard/change_sender_cdc_stream.cpp @@ -270,9 +270,9 @@ class TCdcChangeSenderMain: public TActorBootstrapped<TCdcChangeSenderMain> TPartitionKeyRange KeyRange; struct TLess { - TConstArrayRef<NScheme::TTypeId> Schema; + TConstArrayRef<NScheme::TTypeInfo> Schema; - TLess(const TVector<NScheme::TTypeId>& schema) + TLess(const TVector<NScheme::TTypeInfo>& schema) : Schema(schema) { } @@ -323,7 +323,7 @@ class TCdcChangeSenderMain: public TActorBootstrapped<TCdcChangeSenderMain> }; // TPartitionInfo - TVector<NScheme::TTypeId> Schema; + TVector<NScheme::TTypeInfo> Schema; TVector<TPartitionInfo> Partitions; }; // TKeyDesc @@ -550,7 +550,8 @@ class TCdcChangeSenderMain: public TActorBootstrapped<TCdcChangeSenderMain> KeyDesc->Schema.reserve(pqConfig.PartitionKeySchemaSize()); for (const auto& keySchema : pqConfig.GetPartitionKeySchema()) { - KeyDesc->Schema.push_back(keySchema.GetTypeId()); + // TODO: support pg types + KeyDesc->Schema.push_back(NScheme::TTypeInfo(keySchema.GetTypeId())); } TSet<TPQPartitionInfo, TPQPartitionInfo::TLess> partitions(KeyDesc->Schema); diff --git a/ydb/core/tx/datashard/check_scheme_tx_unit.cpp b/ydb/core/tx/datashard/check_scheme_tx_unit.cpp index c42c0f5c9b9..2182381c0ba 100644 --- a/ydb/core/tx/datashard/check_scheme_tx_unit.cpp +++ b/ydb/core/tx/datashard/check_scheme_tx_unit.cpp @@ -502,7 +502,8 @@ bool TCheckSchemeTxUnit::CheckAlter(TActiveTransaction *activeTx) if (table.Columns.contains(colId)) { const TUserTable::TUserColumn &column = table.Columns.at(colId); Y_VERIFY(column.Name == col.GetName()); - Y_VERIFY(column.Type == col.GetTypeId()); + // TODO: support pg types + Y_VERIFY(column.Type.GetTypeId() == col.GetTypeId()); Y_VERIFY(col.HasFamily()); } } diff --git a/ydb/core/tx/datashard/datashard.cpp b/ydb/core/tx/datashard/datashard.cpp index 887d71410d3..9a9b30fbe69 100644 --- a/ydb/core/tx/datashard/datashard.cpp +++ b/ydb/core/tx/datashard/datashard.cpp @@ -2761,7 +2761,6 @@ void TDataShard::ResolveTablePath(const TActorContext &ctx) void TDataShard::SerializeHistogram(const TUserTable &tinfo, const NTable::THistogram &histogram, - const NScheme::TTypeRegistry &typeRegistry, NKikimrTxDataShard::TEvGetDataHistogramResponse::THistogram &hist) { for (auto &item : histogram) { @@ -2770,15 +2769,13 @@ void TDataShard::SerializeHistogram(const TUserTable &tinfo, TSerializedCellVec key(item.EndKey); for (ui32 ki = 0; ki < tinfo.KeyColumnIds.size(); ++ki) { - NScheme::ITypeSP t = typeRegistry.GetType(tinfo.KeyColumnTypes[ki]); - DbgPrintValue(*rec.AddKeyValues(), key.GetCells()[ki], t.GetTypeId()); + DbgPrintValue(*rec.AddKeyValues(), key.GetCells()[ki], tinfo.KeyColumnTypes[ki]); } } } void TDataShard::SerializeKeySample(const TUserTable &tinfo, const NTable::TKeyAccessSample &keySample, - const NScheme::TTypeRegistry &typeRegistry, NKikimrTxDataShard::TEvGetDataHistogramResponse::THistogram &hist) { THashMap<TString, ui64> accessCounts; @@ -2794,8 +2791,7 @@ void TDataShard::SerializeKeySample(const TUserTable &tinfo, TSerializedCellVec key(item.first); for (ui32 ki = 0; ki < tinfo.KeyColumnIds.size() && ki < key.GetCells().size(); ++ki) { - NScheme::ITypeSP t = typeRegistry.GetType(tinfo.KeyColumnTypes[ki]); - DbgPrintValue(*rec.AddKeyValues(), key.GetCells()[ki], t.GetTypeId()); + DbgPrintValue(*rec.AddKeyValues(), key.GetCells()[ki], tinfo.KeyColumnTypes[ki]); } } Sort(hist.MutableItems()->begin(), hist.MutableItems()->end(), @@ -2867,7 +2863,6 @@ void TDataShard::Handle(TEvDataShard::TEvGetDataHistogramRequest::TPtr &ev, } } - auto ® = *AppData(ctx)->TypeRegistry; for (const auto &pr : TableInfos) { const auto &tinfo = *pr.second; const NTable::TStats &stats = tinfo.Stats.DataStats; @@ -2876,9 +2871,9 @@ void TDataShard::Handle(TEvDataShard::TEvGetDataHistogramRequest::TPtr &ev, hist.SetTableName(pr.second->Name); for (ui32 ki : tinfo.KeyColumnIds) hist.AddKeyNames(tinfo.Columns.FindPtr(ki)->Name); - SerializeHistogram(tinfo, stats.DataSizeHistogram, reg, *hist.MutableSizeHistogram()); - SerializeHistogram(tinfo, stats.RowCountHistogram, reg, *hist.MutableCountHistogram()); - SerializeKeySample(tinfo, tinfo.Stats.AccessStats, reg, *hist.MutableKeyAccessSample()); + SerializeHistogram(tinfo, stats.DataSizeHistogram, *hist.MutableSizeHistogram()); + SerializeHistogram(tinfo, stats.RowCountHistogram, *hist.MutableCountHistogram()); + SerializeKeySample(tinfo, tinfo.Stats.AccessStats, *hist.MutableKeyAccessSample()); } ctx.Send(ev->Sender, response); diff --git a/ydb/core/tx/datashard/datashard__build_index.cpp b/ydb/core/tx/datashard/datashard__build_index.cpp index dbf29287a1a..b2fc2cf6118 100644 --- a/ydb/core/tx/datashard/datashard__build_index.cpp +++ b/ydb/core/tx/datashard/datashard__build_index.cpp @@ -7,6 +7,7 @@ #include <ydb/core/base/counters.h> #include <ydb/core/scheme/scheme_tablecell.h> #include <ydb/core/tablet_flat/flat_row_state.h> +#include <ydb/core/kqp/common/kqp_types.h> #include <ydb/core/tx/tx_proxy/proxy.h> #include <ydb/core/tx/tx_proxy/upload_rows.h> @@ -20,7 +21,7 @@ namespace NKikimr { namespace NDataShard { using TColumnsTags = THashMap<TString, NTable::TTag>; -using TColumnsTypes = THashMap<TString, NScheme::TTypeId>; +using TColumnsTypes = THashMap<TString, NScheme::TTypeInfo>; using TTypes = TVector<std::pair<TString, Ydb::Type>>; using TTags = TVector<NTable::TTag>; @@ -62,6 +63,13 @@ static TTags BuildTags(const TColumnsTags& allTags, const TVector<TString>& inde return result; } +static void ProtoYdbTypeFromTypeInfo(Ydb::Type* type, const NScheme::TTypeInfo typeInfo) { + if (typeInfo.GetTypeId() == NScheme::NTypeIds::Pg) { + type->mutable_pg_type()->set_oid(NPg::PgTypeIdFromTypeDesc(typeInfo.GetTypeDesc())); + } else { + type->set_type_id((Ydb::Type::PrimitiveTypeId)typeInfo.GetTypeId()); + } +} static std::shared_ptr<TTypes> BuildTypes(const TColumnsTypes& types, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) { auto result = std::make_shared<TTypes>(); @@ -69,13 +77,13 @@ static std::shared_ptr<TTypes> BuildTypes(const TColumnsTypes& types, const TVec for (const auto& colName: indexColumns) { Ydb::Type type; - type.set_type_id(static_cast<Ydb::Type_PrimitiveTypeId>(types.at(colName))); + ProtoYdbTypeFromTypeInfo(&type, types.at(colName)); result->emplace_back(colName, type); } for (const auto& colName: dataColumns) { Ydb::Type type; - type.set_type_id(static_cast<Ydb::Type_PrimitiveTypeId>(types.at(colName))); + ProtoYdbTypeFromTypeInfo(&type, types.at(colName)); result->emplace_back(colName, type); } @@ -201,7 +209,7 @@ class TBuildIndexScan : public TActor<TBuildIndexScan>, public NTable::IScan { const ui32 TargetDataColumnPos; // positon of first data column in target table const TTags KeyColumnIds; - const TVector<NScheme::TTypeId> KeyTypes; + const TVector<NScheme::TTypeInfo> KeyTypes; const TSerializedTableRange TableRange; const TSerializedTableRange RequestedRange; diff --git a/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp b/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp index 04135da2148..7f10ef955e3 100644 --- a/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp +++ b/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp @@ -464,8 +464,8 @@ static TIndexes GetIndexes(const NKikimrTxDataShard::TEvConditionalEraseRowsRequ return result; } -static bool CheckUnit(NScheme::TTypeId type, NKikimrSchemeOp::TTTLSettings::EUnit unit, TString& error) { - switch (type) { +static bool CheckUnit(NScheme::TTypeInfo type, NKikimrSchemeOp::TTTLSettings::EUnit unit, TString& error) { + switch (type.GetTypeId()) { case NScheme::NTypeIds::Date: case NScheme::NTypeIds::Datetime: case NScheme::NTypeIds::Timestamp: @@ -496,7 +496,7 @@ static bool CheckUnit(NScheme::TTypeId type, NKikimrSchemeOp::TTTLSettings::EUni break; default: - error = TStringBuilder() << "Unsupported type: " << static_cast<ui32>(type); + error = TStringBuilder() << "Unsupported type: " << static_cast<ui32>(type.GetTypeId()); return false; } } diff --git a/ydb/core/tx/datashard/datashard__engine_host.cpp b/ydb/core/tx/datashard/datashard__engine_host.cpp index ddc9e83714c..825422d3d58 100644 --- a/ydb/core/tx/datashard/datashard__engine_host.cpp +++ b/ydb/core/tx/datashard/datashard__engine_host.cpp @@ -26,7 +26,7 @@ using namespace NTabletFlatExecutor; namespace { NUdf::TUnboxedValue CreateRow(const TVector<TCell>& inRow, - const TVector<NScheme::TTypeId>& inType, + const TVector<NScheme::TTypeInfo>& inType, const THolderFactory& holderFactory) { NUdf::TUnboxedValue* rowItems = nullptr; auto row = holderFactory.CreateDirectArrayHolder(inRow.size(), rowItems); @@ -43,10 +43,10 @@ NUdf::TUnboxedValue CreateRow(const TVector<TCell>& inRow, /// struct ItemInfo { ui32 ColumnId; - NScheme::TTypeId SchemeType; + NScheme::TTypeInfo SchemeType; TOptionalType* OptType; - ItemInfo(ui32 colId, NScheme::TTypeId schemeType, TOptionalType* optType) + ItemInfo(ui32 colId, NScheme::TTypeInfo schemeType, TOptionalType* optType) : ColumnId(colId) , SchemeType(schemeType) , OptType(optType) @@ -88,7 +88,7 @@ struct TRowResultInfo { // reorder columns TVector<TCell> outRow(Reserve(ItemInfos.size())); - TVector<NScheme::TTypeId> outTypes(Reserve(ItemInfos.size())); + TVector<NScheme::TTypeInfo> outTypes(Reserve(ItemInfos.size())); for (ui32 i = 0; i < ItemInfos.size(); ++i) { ui32 colId = ItemInfos[i].ColumnId; outRow.emplace_back(std::move(inRow[colId])); @@ -100,93 +100,6 @@ struct TRowResultInfo { }; /// -struct TRangeResultInfo { - TStructType* ResultType; - TStructType* RowType; - TSmallVec<ItemInfo> ItemInfos; - mutable ui64 Bytes = 0; - TDefaultListRepresentation Rows; - TString FirstKey; - - // optimisation: reuse vectors - TVector<TCell> TmpRow; - TVector<NScheme::TTypeId> TmpTypes; - - TListType* RowsListType() const { return AS_TYPE(TListType, ResultType->GetMemberType(0)); } - TDataType* TruncType() const { return AS_TYPE(TDataType, ResultType->GetMemberType(1)); } - - TRangeResultInfo(const TStructLiteral* columnIds, const THashMap<ui32, TSysTables::TTableColumnInfo>& columns, - TStructType* returnType) - { - ResultType = AS_TYPE(TStructType, returnType); - Y_VERIFY_DEBUG(ResultType->GetMembersCount() == 2); - Y_VERIFY_DEBUG(ResultType->GetMemberName(0) == "List"); - Y_VERIFY_DEBUG(ResultType->GetMemberName(1) == "Truncated"); - - RowType = AS_TYPE(TStructType, RowsListType()->GetItemType()); - - ItemInfos.reserve(columnIds->GetValuesCount()); - for (ui32 i = 0; i < columnIds->GetValuesCount(); ++i) { - TOptionalType * optType = AS_TYPE(TOptionalType, RowType->GetMemberType(i)); - TDataLiteral* literal = AS_VALUE(TDataLiteral, columnIds->GetValue(i)); - ui32 colId = literal->AsValue().Get<ui32>(); - - const TSysTables::TTableColumnInfo * colInfo = columns.FindPtr(colId); - Y_VERIFY(colInfo && (colInfo->Id == colId), "No column info for column"); - ItemInfos.emplace_back(ItemInfo(colId, colInfo->PType, optType)); - } - } - - static TString Serialize(const TVector<TCell>& row, const TVector<NScheme::TTypeId>& types) { - Y_VERIFY(row.size() == types.size()); - - ui32 count = row.size(); - TString str((const char*)&count, sizeof(ui32)); - str.append((const char*)&types[0], count * sizeof(NScheme::TTypeId)); - - TConstArrayRef<TCell> array(&row[0], row.size()); - return str + TSerializedCellVec::Serialize(array); - } - - void AppendRow(const TVector<TCell>& inRow, const THolderFactory& holderFactory) { - if (inRow.empty()) - return; - - Y_VERIFY(inRow.size() >= ItemInfos.size()); - - TmpRow.clear(); - TmpTypes.clear(); - TmpRow.reserve(ItemInfos.size()); - TmpTypes.reserve(ItemInfos.size()); - - for (ui32 i = 0; i < ItemInfos.size(); ++i) { - ui32 colId = ItemInfos[i].ColumnId; - TmpRow.push_back(inRow[colId]); - TmpTypes.push_back(ItemInfos[i].SchemeType); - } - - Bytes += 8; // per row overhead - Rows = Rows.Append(CreateRow(TmpRow, TmpTypes, holderFactory)); - - if (!FirstKey) { - FirstKey = Serialize(TmpRow, TmpTypes); - } - } - - NUdf::TUnboxedValue CreateResult(const THolderFactory& holderFactory) { - NUdf::TUnboxedValue* resultItems = nullptr; - auto result = holderFactory.CreateDirectArrayHolder(4, resultItems); - - resultItems[0] = holderFactory.CreateDirectListHolder(std::move(Rows)); - resultItems[1] = NUdf::TUnboxedValuePod(false); - resultItems[2] = MakeString(FirstKey); - resultItems[3] = NUdf::TUnboxedValuePod(Bytes); - - return std::move(result); - } -}; - -/// class TDataShardSysTable { public: using TUpdateCommand = IEngineFlatHost::TUpdateCommand; @@ -249,7 +162,7 @@ private: const TTableId TableId; TDataShard* Self; THashMap<ui32, TSysTables::TTableColumnInfo> Columns; - TVector<ui32> KeyTypes; + TVector<NScheme::TTypeInfo> KeyTypes; }; @@ -797,7 +710,7 @@ TEngineBay::~TEngineBay() { } void TEngineBay::AddReadRange(const TTableId& tableId, const TVector<NTable::TColumn>& columns, - const TTableRange& range, const TVector<NScheme::TTypeId>& keyTypes, ui64 itemsLimit, bool reverse) + const TTableRange& range, const TVector<NScheme::TTypeInfo>& keyTypes, ui64 itemsLimit, bool reverse) { TVector<TKeyDesc::TColumnOp> columnOps; columnOps.reserve(columns.size()); @@ -821,7 +734,7 @@ void TEngineBay::AddReadRange(const TTableId& tableId, const TVector<NTable::TCo } void TEngineBay::AddWriteRange(const TTableId& tableId, const TTableRange& range, - const TVector<NScheme::TTypeId>& keyTypes, const TVector<TColumnWriteMeta>& columns, + const TVector<NScheme::TTypeInfo>& keyTypes, const TVector<TColumnWriteMeta>& columns, bool isPureEraseOp) { TVector<TKeyDesc::TColumnOp> columnOps; diff --git a/ydb/core/tx/datashard/datashard__engine_host.h b/ydb/core/tx/datashard/datashard__engine_host.h index 06ddf773c14..588d7d940c0 100644 --- a/ydb/core/tx/datashard/datashard__engine_host.h +++ b/ydb/core/tx/datashard/datashard__engine_host.h @@ -66,9 +66,9 @@ public: } void AddReadRange(const TTableId& tableId, const TVector<NTable::TColumn>& columns, const TTableRange& range, - const TVector<NScheme::TTypeId>& keyTypes, ui64 itemsLimit = 0, bool reverse = false); + const TVector<NScheme::TTypeInfo>& keyTypes, ui64 itemsLimit = 0, bool reverse = false); - void AddWriteRange(const TTableId& tableId, const TTableRange& range, const TVector<NScheme::TTypeId>& keyTypes, + void AddWriteRange(const TTableId& tableId, const TTableRange& range, const TVector<NScheme::TTypeInfo>& keyTypes, const TVector<TColumnWriteMeta>& columns, bool isPureEraseOp); void MarkTxLoaded() { diff --git a/ydb/core/tx/datashard/datashard__kqp_scan.cpp b/ydb/core/tx/datashard/datashard__kqp_scan.cpp index 8b993322d40..95f172421c3 100644 --- a/ydb/core/tx/datashard/datashard__kqp_scan.cpp +++ b/ydb/core/tx/datashard/datashard__kqp_scan.cpp @@ -59,7 +59,7 @@ public: { if (DataFormat == NKikimrTxDataShard::EScanDataFormat::ARROW) { BatchBuilder = MakeHolder<NArrow::TArrowBatchBuilder>(); - TVector<std::pair<TString, NScheme::TTypeId>> schema; + TVector<std::pair<TString, NScheme::TTypeInfo>> schema; if (!Tags.empty()) { Types.reserve(Tags.size()); schema.reserve(Tags.size()); @@ -511,7 +511,7 @@ private: const TSmallVec<TSerializedTableRange> TableRanges; ui32 CurrentRange; const TSmallVec<NTable::TTag> Tags; - TSmallVec<NScheme::TTypeId> Types; + TSmallVec<NScheme::TTypeInfo> Types; const TSmallVec<bool> SkipNullKeys; const NYql::NDqProto::EDqStatsMode StatsMode; const TInstant Deadline; @@ -575,7 +575,6 @@ void TDataShard::Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext& return; } - for (int i = 0; i < request.GetColumnTags().size(); ++i) { auto* column = tableColumns.FindPtr(request.GetColumnTags(i)); if (!column) { @@ -585,7 +584,8 @@ void TDataShard::Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext& return; } - if (column->Type != request.GetColumnTypes(i)) { + // TODO: support pg types + if (column->Type.GetTypeId() != request.GetColumnTypes(i)) { reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "." << " Table '" << request.GetTablePath() << "'" << " column " << request.GetColumnTags(i) << " type mismatch at " << TabletID()); diff --git a/ydb/core/tx/datashard/datashard__read_columns.cpp b/ydb/core/tx/datashard/datashard__read_columns.cpp index 6b7200f552b..983abe32f79 100644 --- a/ydb/core/tx/datashard/datashard__read_columns.cpp +++ b/ydb/core/tx/datashard/datashard__read_columns.cpp @@ -42,7 +42,7 @@ class TReadColumnsScan : public INoTxScan { const TKeyBoundary From; const TKeyBoundary To; const TVector<NTable::TTag> ValueColumns; - const TVector<NScheme::TTypeId> ValueColumnTypes; + const TVector<NScheme::TTypeInfo> ValueColumnTypes; const ui64 RowsLimit = 100000; const ui64 BytesLimit = 1024*1024; const TKeyBoundary ShardEnd; @@ -62,7 +62,7 @@ public: TReadColumnsScan(const TKeyBoundary& keyFrom, const TKeyBoundary& keyTo, const TVector<NTable::TTag>& valueColumns, - const TVector<NScheme::TTypeId> valueColumnTypes, + const TVector<NScheme::TTypeInfo> valueColumnTypes, std::unique_ptr<IBlockBuilder>&& blockBuilder, ui64 rowsLimit, ui64 bytesLimit, const TKeyBoundary& shardEnd, @@ -342,8 +342,8 @@ public: } TVector<NTable::TTag> valueColumns; - TVector<NScheme::TTypeId> valueColumnTypes; - TVector<std::pair<TString, NScheme::TTypeId>> columns; + TVector<NScheme::TTypeInfo> valueColumnTypes; + TVector<std::pair<TString, NScheme::TTypeInfo>> columns; if (Ev->Get()->Record.GetColumns().empty()) { SetError(NKikimrTxDataShard::TError::BAD_ARGUMENT, "Empty column list"); diff --git a/ydb/core/tx/datashard/datashard__read_iterator.cpp b/ydb/core/tx/datashard/datashard__read_iterator.cpp index f33e816d9c4..6b35b2b68ed 100644 --- a/ydb/core/tx/datashard/datashard__read_iterator.cpp +++ b/ydb/core/tx/datashard/datashard__read_iterator.cpp @@ -20,7 +20,7 @@ constexpr ui64 MinRowsPerCheck = 1000; class TCellBlockBuilder : public IBlockBuilder { public: bool Start( - const TVector<std::pair<TString, NScheme::TTypeId>>& columns, + const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns, ui64 maxRowsInBlock, ui64 maxBytesInBlock, TString& err) override @@ -49,7 +49,7 @@ public: TVector<TOwnedCellVec> FlushBatch() { return std::move(Rows); } private: - TVector<std::pair<TString, NScheme::TTypeId>> Columns; + TVector<std::pair<TString, NScheme::TTypeInfo>> Columns; TVector<TOwnedCellVec> Rows; ui64 BytesCount = 0; @@ -61,10 +61,10 @@ private: struct TShortColumnInfo { NTable::TTag Tag; - NScheme::TTypeId Type; + NScheme::TTypeInfo Type; TString Name; - TShortColumnInfo(NTable::TTag tag, NScheme::TTypeId type, const TString& name) + TShortColumnInfo(NTable::TTag tag, NScheme::TTypeInfo type, const TString& name) : Tag(tag) , Type(type) , Name(name) @@ -72,7 +72,8 @@ struct TShortColumnInfo { TString Dump() const { TStringStream ss; - ss << "{Tag: " << Tag << ", Type: " << Type << ", Name: " << Name << "}"; + // TODO: support pg types + ss << "{Tag: " << Tag << ", Type: " << Type.GetTypeId() << ", Name: " << Name << "}"; return ss.Str(); } }; @@ -98,13 +99,13 @@ struct TShortTableInfo { KeyColumnCount = schema.Keys->Types.size(); KeyColumnTypes.reserve(KeyColumnCount); for (auto type: schema.Keys->Types) { - KeyColumnTypes.push_back(type.GetTypeId()); + KeyColumnTypes.push_back(type.ToTypeInfo()); } // note that we don't have column names here, but // for cellvec we will not need them at all for (const auto& col: schema.Cols) { - Columns.emplace(col.Tag, TShortColumnInfo(col.Tag, col.TypeId, "")); + Columns.emplace(col.Tag, TShortColumnInfo(col.Tag, col.TypeInfo, "")); } } @@ -123,15 +124,15 @@ struct TShortTableInfo { ui32 LocalTid = 0; ui64 SchemaVersion = 0; size_t KeyColumnCount = 0; - TVector<NScheme::TTypeId> KeyColumnTypes; + TVector<NScheme::TTypeInfo> KeyColumnTypes; TMap<NTable::TTag, TShortColumnInfo> Columns; }; -TVector<std::pair<TString, NScheme::TTypeId>> GetNameTypeColumns( +TVector<std::pair<TString, NScheme::TTypeInfo>> GetNameTypeColumns( const std::vector<NTable::TTag>& tags, const TShortTableInfo& tableInfo) { - TVector<std::pair<TString, NScheme::TTypeId>> result; + TVector<std::pair<TString, NScheme::TTypeInfo>> result; for (auto tag: tags) { auto it = tableInfo.Columns.find(tag); if (it == tableInfo.Columns.end()) { @@ -227,7 +228,7 @@ class TReader { const TShortTableInfo& TableInfo; TDataShard* Self; - std::vector<NKikimr::NScheme::TTypeId> ColumnTypes; + std::vector<NScheme::TTypeInfo> ColumnTypes; ui32 FirstUnprocessedQuery; TString LastProcessedKey; @@ -1305,7 +1306,7 @@ private: void PrepareValidationInfo(const TActorContext&, const TReadIteratorState& state) { TTableId tableId(state.PathId.OwnerId, state.PathId.LocalPathId, state.SchemaVersion); - TVector<NScheme::TTypeId> keyTypes; + TVector<NScheme::TTypeInfo> keyTypes; TVector<TKeyDesc::TColumnOp> columnOps; columnOps.reserve(TableInfo.Columns.size()); diff --git a/ydb/core/tx/datashard/datashard__s3.cpp b/ydb/core/tx/datashard/datashard__s3.cpp index f5dfe0d65e7..fa3b7d94f6f 100644 --- a/ydb/core/tx/datashard/datashard__s3.cpp +++ b/ydb/core/tx/datashard/datashard__s3.cpp @@ -163,7 +163,7 @@ public: prefixColumns.GetCells().size()), "Unexpected out of range key returned from iterator"); - Y_VERIFY(currentKey.Types[pathColPos] == NScheme::NTypeIds::Utf8); + Y_VERIFY(currentKey.Types[pathColPos].GetTypeId() == NScheme::NTypeIds::Utf8); const TCell& pathCell = currentKey.Cells()[pathColPos]; TString path = TString((const char*)pathCell.Data(), pathCell.Size()); diff --git a/ydb/core/tx/datashard/datashard_change_receiving.cpp b/ydb/core/tx/datashard/datashard_change_receiving.cpp index 0cc14c5f178..eb7ab505929 100644 --- a/ydb/core/tx/datashard/datashard_change_receiving.cpp +++ b/ydb/core/tx/datashard/datashard_change_receiving.cpp @@ -188,7 +188,7 @@ class TDataShard::TTxApplyChangeRecords: public TTransactionBase<TDataShard> { const auto type = tableInfo.KeyColumnTypes.at(i); const auto& cell = KeyCells.GetCells().at(i); - if (type == NScheme::NTypeIds::Uint8 && !cell.IsNull() && cell.AsValue<ui8>() > 127) { + if (type.GetTypeId() == NScheme::NTypeIds::Uint8 && !cell.IsNull() && cell.AsValue<ui8>() > 127) { AddRecordStatus(ctx, record.GetOrder(), NKikimrChangeExchange::TEvStatus::STATUS_REJECT, NKikimrChangeExchange::TEvStatus::REASON_SCHEME_ERROR, "Keys with Uint8 column values >127 are currently prohibited"); diff --git a/ydb/core/tx/datashard/datashard_common_upload.cpp b/ydb/core/tx/datashard/datashard_common_upload.cpp index dd138f7de82..3501f8d7cb2 100644 --- a/ydb/core/tx/datashard/datashard_common_upload.cpp +++ b/ydb/core/tx/datashard/datashard_common_upload.cpp @@ -72,7 +72,7 @@ bool TCommonUploadOps<TEvRequest, TEvResponse>::Execute(TDataShard* self, TTrans // Prepare (id, Type) vector for value columns TVector<NTable::TTag> tagsForSelect; - TVector<std::pair<ui32, NScheme::TTypeId>> valueCols; + TVector<std::pair<ui32, NScheme::TTypeInfo>> valueCols; for (const auto& colId : record.GetRowScheme().GetValueColumnIds()) { if (readForTableShadow) { tagsForSelect.push_back(colId); @@ -114,7 +114,7 @@ bool TCommonUploadOps<TEvRequest, TEvResponse>::Execute(TDataShard* self, TTrans ui64 keyBytes = 0; for (const auto& kt : tableInfo.KeyColumnTypes) { const TCell& c = keyCells.GetCells()[ki]; - if (kt == NScheme::NTypeIds::Uint8 && !c.IsNull() && c.AsValue<ui8>() > 127) { + if (kt.GetTypeId() == NScheme::NTypeIds::Uint8 && !c.IsNull() && c.AsValue<ui8>() > 127) { SetError(NKikimrTxDataShard::TError::BAD_ARGUMENT, "Keys with Uint8 column values >127 are currently prohibited"); return true; } diff --git a/ydb/core/tx/datashard/datashard_direct_erase.cpp b/ydb/core/tx/datashard/datashard_direct_erase.cpp index 054bedfc2b8..2014d71bc2a 100644 --- a/ydb/core/tx/datashard/datashard_direct_erase.cpp +++ b/ydb/core/tx/datashard/datashard_direct_erase.cpp @@ -98,7 +98,7 @@ TDirectTxErase::EStatus TDirectTxErase::CheckedExecute( const auto& kt = tableInfo.KeyColumnTypes[ki]; const TCell& cell = keyCells.GetCells()[ki]; - if (kt == NScheme::NTypeIds::Uint8 && !cell.IsNull() && cell.AsValue<ui8>() > 127) { + if (kt.GetTypeId() == NScheme::NTypeIds::Uint8 && !cell.IsNull() && cell.AsValue<ui8>() > 127) { status = NKikimrTxDataShard::TEvEraseRowsResponse::BAD_REQUEST; error = "Keys with Uint8 column values >127 are currently prohibited"; return EStatus::Error; diff --git a/ydb/core/tx/datashard/datashard_distributed_erase.cpp b/ydb/core/tx/datashard/datashard_distributed_erase.cpp index e7a15a9eabd..4c14de3c1d7 100644 --- a/ydb/core/tx/datashard/datashard_distributed_erase.cpp +++ b/ydb/core/tx/datashard/datashard_distributed_erase.cpp @@ -74,8 +74,8 @@ class TDistEraser: public TActorBootstrapped<TDistEraser> { }); } - static TVector<NUdf::TDataTypeId> MakeKeyColumnTypes(const TNavigate::TEntry& entry) { - return MakeKeyColumnSmth<NUdf::TDataTypeId>(entry, [](auto column) { + static TVector<NScheme::TTypeInfo> MakeKeyColumnTypes(const TNavigate::TEntry& entry) { + return MakeKeyColumnSmth<NScheme::TTypeInfo>(entry, [](auto column) { return column.PType; }); } @@ -87,7 +87,7 @@ class TDistEraser: public TActorBootstrapped<TDistEraser> { } static THolder<TKeyDesc> MakeKeyDesc(const TNavigate::TEntry& entry) { - const TVector<NUdf::TDataTypeId> keyColumnTypes = MakeKeyColumnTypes(entry); + const TVector<NScheme::TTypeInfo> keyColumnTypes = MakeKeyColumnTypes(entry); return MakeHolder<TKeyDesc>( entry.TableId, GetFullRange(keyColumnTypes.size()).ToTableRange(), diff --git a/ydb/core/tx/datashard/datashard_impl.h b/ydb/core/tx/datashard/datashard_impl.h index c2332ff3b9a..2ec648f65af 100644 --- a/ydb/core/tx/datashard/datashard_impl.h +++ b/ydb/core/tx/datashard/datashard_impl.h @@ -2655,11 +2655,9 @@ protected: bool OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev, const TActorContext &ctx) override; void SerializeHistogram(const TUserTable &tinfo, const NTable::THistogram &histogram, - const NScheme::TTypeRegistry &typeRegistry, NKikimrTxDataShard::TEvGetDataHistogramResponse::THistogram &hist); void SerializeKeySample(const TUserTable &tinfo, const NTable::TKeyAccessSample &keySample, - const NScheme::TTypeRegistry &typeRegistry, NKikimrTxDataShard::TEvGetDataHistogramResponse::THistogram &hist); bool ByKeyFilterDisabled() const; diff --git a/ydb/core/tx/datashard/datashard_kqp.cpp b/ydb/core/tx/datashard/datashard_kqp.cpp index 92fbe03e4b7..01c51025612 100644 --- a/ydb/core/tx/datashard/datashard_kqp.cpp +++ b/ydb/core/tx/datashard/datashard_kqp.cpp @@ -4,6 +4,7 @@ #include <ydb/core/kqp/kqp.h> #include <ydb/core/kqp/runtime/kqp_tasks_runner.h> #include <ydb/core/scheme/scheme_tablecell.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/tx/datashard/datashard_locks.h> #include <ydb/core/tx/datashard/datashard_user_table.h> #include <ydb/core/tx/datashard/range_ops.h> @@ -300,7 +301,9 @@ using TWriteOpMeta = NKikimrTxDataShard::TKqpTransaction::TDataTaskMeta::TWriteO using TColumnMeta = NKikimrTxDataShard::TKqpTransaction::TColumnMeta; NTable::TColumn GetColumn(const TColumnMeta& columnMeta) { - return NTable::TColumn(columnMeta.GetName(), columnMeta.GetId(), columnMeta.GetType()); + auto typeInfo = NScheme::TypeInfoFromProtoColumnType(columnMeta.GetType(), + columnMeta.HasTypeInfo() ? &columnMeta.GetTypeInfo() : nullptr); + return NTable::TColumn(columnMeta.GetName(), columnMeta.GetId(), typeInfo); } TVector<NTable::TColumn> GetColumns(const TReadOpMeta& readMeta) { @@ -451,9 +454,11 @@ void KqpSetTxLocksKeys(const NKikimrTxDataShard::TKqpLocks& locks, const TSysLoc } static TTableId sysLocksTableId = TTableId(TSysTables::SysSchemeShard, TSysTables::SysTableLocks2); - static TVector<NScheme::TTypeId> lockRowType = { - NScheme::TUint64::TypeId, NScheme::TUint64::TypeId, - NScheme::TUint64::TypeId, NScheme::TUint64::TypeId, + static TVector<NScheme::TTypeInfo> lockRowType = { + NScheme::TTypeInfo(NScheme::TUint64::TypeId), + NScheme::TTypeInfo(NScheme::TUint64::TypeId), + NScheme::TTypeInfo(NScheme::TUint64::TypeId), + NScheme::TTypeInfo(NScheme::TUint64::TypeId), }; for (auto& lock : locks.GetLocks()) { diff --git a/ydb/core/tx/datashard/datashard_kqp_compute.cpp b/ydb/core/tx/datashard/datashard_kqp_compute.cpp index 50662025ef1..8bd347f5b61 100644 --- a/ydb/core/tx/datashard/datashard_kqp_compute.cpp +++ b/ydb/core/tx/datashard/datashard_kqp_compute.cpp @@ -100,14 +100,14 @@ const NDataShard::TUserTable::TUserColumn& TKqpDatashardComputeContext::GetKeyCo return col; } -THashMap<TString, NScheme::TTypeId> TKqpDatashardComputeContext::GetKeyColumnsMap(const TTableId &tableId) const { +THashMap<TString, NScheme::TTypeInfo> TKqpDatashardComputeContext::GetKeyColumnsMap(const TTableId &tableId) const { MKQL_ENSURE_S(Shard); const NDataShard::TUserTable::TCPtr* tablePtr = Shard->GetUserTables().FindPtr(tableId.PathId.LocalPathId); MKQL_ENSURE_S(tablePtr); const NDataShard::TUserTable::TCPtr table = *tablePtr; MKQL_ENSURE_S(table); - THashMap<TString, NScheme::TTypeId> columnsMap; + THashMap<TString, NScheme::TTypeInfo> columnsMap; for (size_t i = 0 ; i < table->KeyColumnTypes.size(); i++) { auto col = table->Columns.at(table->KeyColumnIds[i]); MKQL_ENSURE_S(col.IsKey); @@ -425,7 +425,7 @@ bool TKqpDatashardComputeContext::ReadRow(const TTableId& tableId, TArrayRef<con MKQL_ENSURE_S(columnTags.size() == dbRow.Size(), "Invalid local db row size."); - TVector<NScheme::TTypeId> types(columnTags.size()); + TVector<NScheme::TTypeInfo> types(columnTags.size()); for (size_t i = 0; i < columnTags.size(); ++i) { types[i] = tableInfo->Columns.at(columnTags[i]).PType; } diff --git a/ydb/core/tx/datashard/datashard_kqp_compute.h b/ydb/core/tx/datashard/datashard_kqp_compute.h index 5e0fc52e70c..de6d81372ab 100644 --- a/ydb/core/tx/datashard/datashard_kqp_compute.h +++ b/ydb/core/tx/datashard/datashard_kqp_compute.h @@ -35,7 +35,7 @@ public: const NDataShard::TUserTable::TUserColumn& GetKeyColumnInfo( const NDataShard::TUserTable& table, ui32 keyIndex) const; - THashMap<TString, NScheme::TTypeId> GetKeyColumnsMap(const TTableId &tableId) const; + THashMap<TString, NScheme::TTypeInfo> GetKeyColumnsMap(const TTableId &tableId) const; void SetHasPersistentChannels(bool value) { PersistentChannels = value; } bool HasPersistentChannels() const { return PersistentChannels; } diff --git a/ydb/core/tx/datashard/datashard_kqp_delete_rows.cpp b/ydb/core/tx/datashard/datashard_kqp_delete_rows.cpp index f03e13ddf39..f73be2d14a2 100644 --- a/ydb/core/tx/datashard/datashard_kqp_delete_rows.cpp +++ b/ydb/core/tx/datashard/datashard_kqp_delete_rows.cpp @@ -94,7 +94,7 @@ public: public: TKqpDeleteRowsWrapper(TComputationMutables& mutables, TKqpDatashardComputeContext& computeCtx, - const TTableId& tableId, IComputationNode* rowsNode, TVector<NUdf::TDataTypeId> rowTypes, TVector<ui32> keyIndices, const TTypeEnvironment& env) + const TTableId& tableId, IComputationNode* rowsNode, TVector<NScheme::TTypeInfo> rowTypes, TVector<ui32> keyIndices, const TTypeEnvironment& env) : TBase(mutables) , TableId(tableId) , RowsNode(rowsNode) @@ -112,7 +112,7 @@ private: private: TTableId TableId; IComputationNode* RowsNode; - const TVector<NUdf::TDataTypeId> RowTypes; + const TVector<NScheme::TTypeInfo> RowTypes; const TVector<ui32> KeyIndices; const TTypeEnvironment& Env; TKqpTableStats& ShardTableStats; @@ -139,7 +139,7 @@ IComputationNode* WrapKqpDeleteRows(TCallable& callable, const TComputationNodeF << ", actual: " << rowType->GetMembersCount()); THashMap<TString, ui32> inputIndex; - TVector<NUdf::TDataTypeId> rowTypes(rowType->GetMembersCount()); + TVector<NScheme::TTypeInfo> rowTypes(rowType->GetMembersCount()); for (ui32 i = 0; i < rowType->GetMembersCount(); ++i) { const auto& name = rowType->GetMemberName(i); MKQL_ENSURE_S(inputIndex.emplace(TString(name), i).second); @@ -149,7 +149,8 @@ IComputationNode* WrapKqpDeleteRows(TCallable& callable, const TComputationNodeF ? AS_TYPE(TDataType, AS_TYPE(TOptionalType, memberType)->GetItemType())->GetSchemeType() : AS_TYPE(TDataType, memberType)->GetSchemeType(); - rowTypes[i] = typeId; + // TODO: support pg types + rowTypes[i] = NScheme::TTypeInfo(typeId); } TVector<ui32> keyIndices(tableInfo->KeyColumnIds.size()); @@ -160,8 +161,8 @@ IComputationNode* WrapKqpDeleteRows(TCallable& callable, const TComputationNodeF MKQL_ENSURE_S(rowTypes[it->second] == columnInfo.Type, "Key type mismatch" << ", column: " << columnInfo.Name - << ", expected: " << columnInfo.Type - << ", actual: " << rowTypes[it->second]); + << ", expected: " << NScheme::TypeName(columnInfo.Type) + << ", actual: " << NScheme::TypeName(rowTypes[it->second])); keyIndices[i] = it->second; } diff --git a/ydb/core/tx/datashard/datashard_kqp_lookup_table.cpp b/ydb/core/tx/datashard/datashard_kqp_lookup_table.cpp index 8cd0d4de4a8..eb50e43406e 100644 --- a/ydb/core/tx/datashard/datashard_kqp_lookup_table.cpp +++ b/ydb/core/tx/datashard/datashard_kqp_lookup_table.cpp @@ -17,14 +17,14 @@ struct TParseLookupTableResult { TTableId TableId; TRuntimeNode LookupKeys; TVector<ui32> KeyIndices; - TVector<NUdf::TDataTypeId> KeyTypes; + TVector<NScheme::TTypeInfo> KeyTypes; TSmallVec<NTable::TTag> Columns; TSmallVec<NTable::TTag> SystemColumns; TSmallVec<bool> SkipNullKeys; }; -void ValidateLookupKeys(const TType* inputType, const THashMap<TString, NScheme::TTypeId>& keyColumns) { +void ValidateLookupKeys(const TType* inputType, const THashMap<TString, NScheme::TTypeInfo>& keyColumns) { MKQL_ENSURE_S(inputType); auto rowType = AS_TYPE(TStructType, AS_TYPE(TStreamType, inputType)->GetItemType()); @@ -34,7 +34,9 @@ void ValidateLookupKeys(const TType* inputType, const THashMap<TString, NScheme: auto columnType = keyColumns.FindPtr(name); MKQL_ENSURE_S(columnType); - MKQL_ENSURE_S(dataType == *columnType, "Key column type mismatch, column: " << name); + + // TODO: support pg types + MKQL_ENSURE_S(dataType == columnType->GetTypeId(), "Key column type mismatch, column: " << name); } } @@ -63,12 +65,17 @@ TParseLookupTableResult ParseLookupTable(TCallable& callable) { auto keyTypes = AS_TYPE(TStructType, AS_TYPE(TStreamType, keysNode.GetStaticType())->GetItemType()); result.KeyTypes.resize(keyTypes->GetMembersCount()); for (ui32 i = 0; i < result.KeyTypes.size(); ++i) { + // TODO: support pg types if (keyTypes->GetMemberType(i)->IsOptional()) { - auto type = AS_TYPE(TDataType, AS_TYPE(TOptionalType, keyTypes->GetMemberType(i))->GetItemType()); - result.KeyTypes[i] = type->GetSchemeType(); + auto type = AS_TYPE(TOptionalType, keyTypes->GetMemberType(i))->GetItemType(); + MKQL_ENSURE(type->GetKind() != TType::EKind::Pg, "pg types are not supported"); + auto dataType = AS_TYPE(TDataType, type); + result.KeyTypes[i] = NScheme::TTypeInfo(dataType->GetSchemeType()); } else { - auto type = AS_TYPE(TDataType, keyTypes->GetMemberType(i)); - result.KeyTypes[i] = type->GetSchemeType(); + auto type = keyTypes->GetMemberType(i); + MKQL_ENSURE(type->GetKind() != TType::EKind::Pg, "pg types are not supported"); + auto dataType = AS_TYPE(TDataType, type); + result.KeyTypes[i] = NScheme::TTypeInfo(dataType->GetSchemeType()); } } diff --git a/ydb/core/tx/datashard/datashard_kqp_read_table.cpp b/ydb/core/tx/datashard/datashard_kqp_read_table.cpp index 2054daaadec..bf87ae31b31 100644 --- a/ydb/core/tx/datashard/datashard_kqp_read_table.cpp +++ b/ydb/core/tx/datashard/datashard_kqp_read_table.cpp @@ -16,16 +16,18 @@ using namespace NUdf; namespace { -void ValidateKeyType(const TType* keyType, NScheme::TTypeId expectedType) { +void ValidateKeyType(const TType* keyType, NScheme::TTypeInfo expectedType) { auto type = keyType; if (type->IsOptional()) { type = AS_TYPE(TOptionalType, type)->GetItemType(); } + // TODO: support pg types + MKQL_ENSURE_S(type->GetKind() != TType::EKind::Pg, "pg types are not supported"); auto dataType = AS_TYPE(TDataType, type)->GetSchemeType(); - MKQL_ENSURE_S(dataType == expectedType); + MKQL_ENSURE_S(dataType == expectedType.GetTypeId()); } void ValidateKeyTuple(const TTupleType* tupleType, const NDataShard::TUserTable& tableInfo, @@ -116,7 +118,10 @@ TSerializedTableRange BuildRange(const TTupleType* fromType, const NUdf::TUnboxe } } - cells.emplace_back(MakeCell(AS_TYPE(TDataType, type)->GetSchemeType(), value, typeEnv, /* copy */ true)); + // TODO: support pg types + MKQL_ENSURE_S(type->GetKind() != TType::EKind::Pg, "pg types are not supported"); + auto typeInfo = NScheme::TTypeInfo(AS_TYPE(TDataType, type)->GetSchemeType()); + cells.emplace_back(MakeCell(typeInfo, value, typeEnv, /* copy */ true)); } return cells; diff --git a/ydb/core/tx/datashard/datashard_kqp_upsert_rows.cpp b/ydb/core/tx/datashard/datashard_kqp_upsert_rows.cpp index 8e6e3ba023f..5abccb769c3 100644 --- a/ydb/core/tx/datashard/datashard_kqp_upsert_rows.cpp +++ b/ydb/core/tx/datashard/datashard_kqp_upsert_rows.cpp @@ -63,11 +63,13 @@ public: command.Operation = TKeyDesc::EColumnOperation::Set; auto rowIndex = upsertColumn.RowIndex; - NUdf::TDataTypeId type = Owner.RowTypes[rowIndex]; + NScheme::TTypeInfo type = Owner.RowTypes[rowIndex]; NUdf::TUnboxedValue value = Row.GetElement(rowIndex); if (value) { - auto slot = NUdf::GetDataSlot(type); + // TODO: support pg types + Y_VERIFY(type.GetTypeId() != NScheme::NTypeIds::Pg, "pg types are not supported"); + auto slot = NUdf::GetDataSlot(type.GetTypeId()); MKQL_ENSURE(IsValidValue(slot, value), "Malformed value for type: " << NUdf::GetDataTypeInfo(slot).Name << ", " << value); } @@ -127,7 +129,7 @@ public: public: TKqpUpsertRowsWrapper(TComputationMutables& mutables, TKqpDatashardComputeContext& computeCtx, - const TTableId& tableId, IComputationNode* rowsNode, TVector<NUdf::TDataTypeId>&& rowTypes, + const TTableId& tableId, IComputationNode* rowsNode, TVector<NScheme::TTypeInfo>&& rowTypes, TVector<ui32>&& keyIndices, TVector<TUpsertColumn>&& upsertColumns, const TTypeEnvironment& env) : TBase(mutables) , TableId(tableId) @@ -147,7 +149,7 @@ private: private: TTableId TableId; IComputationNode* RowsNode; - TVector<NUdf::TDataTypeId> RowTypes; + TVector<NScheme::TTypeInfo> RowTypes; TVector<ui32> KeyIndices; TVector<TUpsertColumn> UpsertColumns; const TTypeEnvironment& Env; @@ -176,11 +178,12 @@ IComputationNode* WrapKqpUpsertRows(TCallable& callable, const TComputationNodeF "not enough columns in the runtime node"); THashMap<TStringBuf, ui32> inputIndex; - TVector<NUdf::TDataTypeId> rowTypes(rowType->GetMembersCount()); + TVector<NScheme::TTypeInfo> rowTypes(rowType->GetMembersCount()); for (ui32 i = 0; i < rowTypes.size(); ++i) { const auto& name = rowType->GetMemberName(i); MKQL_ENSURE_S(inputIndex.emplace(name, i).second); - rowTypes[i] = NKqp::UnwrapDataTypeFromStruct(*rowType, i); + // TODO: support pg types + rowTypes[i] = NScheme::TTypeInfo(NKqp::UnwrapDataTypeFromStruct(*rowType, i)); } TVector<ui32> keyIndices(tableInfo->KeyColumnIds.size()); @@ -190,7 +193,8 @@ IComputationNode* WrapKqpUpsertRows(TCallable& callable, const TComputationNodeF auto it = inputIndex.find(columnInfo.Name); MKQL_ENSURE_S(it != inputIndex.end()); auto typeId = NKqp::UnwrapDataTypeFromStruct(*rowType, it->second); - MKQL_ENSURE_S(typeId == columnInfo.Type, "row key type missmatch with table key type"); + // TODO: support pg types + MKQL_ENSURE_S(typeId == columnInfo.Type.GetTypeId(), "row key type missmatch with table key type"); keyIndices[i] = it->second; } diff --git a/ydb/core/tx/datashard/datashard_locks.h b/ydb/core/tx/datashard/datashard_locks.h index 5acb2690e07..d5c910fb4eb 100644 --- a/ydb/core/tx/datashard/datashard_locks.h +++ b/ydb/core/tx/datashard/datashard_locks.h @@ -408,12 +408,12 @@ public: return KeyColumnTypes.size(); } - NScheme::TTypeId GetKeyColumnType(ui32 pos) const { + NScheme::TTypeInfo GetKeyColumnType(ui32 pos) const { Y_VERIFY(pos < KeyColumnTypes.size()); return KeyColumnTypes[pos]; } - void UpdateKeyColumnsTypes(const TVector<NScheme::TTypeId>& keyTypes) { + void UpdateKeyColumnsTypes(const TVector<NScheme::TTypeInfo>& keyTypes) { Y_VERIFY(KeyColumnTypes.size() <= keyTypes.size()); if (KeyColumnTypes.size() < keyTypes.size()) { KeyColumnTypes = keyTypes; @@ -433,7 +433,7 @@ public: private: const TPathId TableId; - TVector<NScheme::TTypeId> KeyColumnTypes; + TVector<NScheme::TTypeInfo> KeyColumnTypes; TRangeTreap<TLockInfo*> Ranges; THashSet<TLockInfo*> ShardLocks; THashSet<TLockInfo*> WriteLocks; diff --git a/ydb/core/tx/datashard/datashard_repl_offsets.cpp b/ydb/core/tx/datashard/datashard_repl_offsets.cpp index 176a6ba0318..5e37ab5a3c8 100644 --- a/ydb/core/tx/datashard/datashard_repl_offsets.cpp +++ b/ydb/core/tx/datashard/datashard_repl_offsets.cpp @@ -57,7 +57,7 @@ namespace NKikimr::NDataShard { const TPathId& pathId, const ui64 id, const TString& name, - const TVector<NScheme::TTypeId>* keyColumnTypes) + const TVector<NScheme::TTypeInfo>* keyColumnTypes) : PathId(pathId) , Id(id) , Name(name) @@ -247,7 +247,7 @@ namespace NKikimr::NDataShard { TReplicatedTableState::TReplicatedTableState( const TPathId& pathId, - const TVector<NScheme::TTypeId>& keyColumnTypes) + const TVector<NScheme::TTypeInfo>& keyColumnTypes) : PathId(pathId) , KeyColumnTypes(keyColumnTypes) { } diff --git a/ydb/core/tx/datashard/datashard_repl_offsets.h b/ydb/core/tx/datashard/datashard_repl_offsets.h index 7377d0c01b6..73c33d46acd 100644 --- a/ydb/core/tx/datashard/datashard_repl_offsets.h +++ b/ydb/core/tx/datashard/datashard_repl_offsets.h @@ -41,7 +41,7 @@ namespace NKikimr::NDataShard { struct TLessByKeyPrefix { // Points to vector in TReplicatedTableState - const TVector<NScheme::TTypeId>* KeyColumnTypes; + const TVector<NScheme::TTypeInfo>* KeyColumnTypes; using is_transparent = void; @@ -72,7 +72,7 @@ namespace NKikimr::NDataShard { const TPathId& pathId, const ui64 id, const TString& name, - const TVector<NScheme::TTypeId>* keyColumnTypes); + const TVector<NScheme::TTypeInfo>* keyColumnTypes); ui64 CalcStatBytes(const TOffsetState& state) const; void AddStatBytes(const TOffsetState& state); @@ -99,14 +99,14 @@ namespace NKikimr::NDataShard { class TReplicatedTableState { public: const TPathId PathId; - TVector<NScheme::TTypeId> KeyColumnTypes; + TVector<NScheme::TTypeInfo> KeyColumnTypes; TMap<ui64, TReplicationSourceState> SourceById; THashMap<TString, TReplicationSourceState*> Sources; ui64 NextSourceId = 1; TReplicatedTableState( const TPathId& pathId, - const TVector<NScheme::TTypeId>& keyColumnTypes); + const TVector<NScheme::TTypeInfo>& keyColumnTypes); TReplicationSourceState* FindSource(ui64 sourceId); TReplicationSourceState& LoadSource(ui64 sourceId, const TString& sourceName); diff --git a/ydb/core/tx/datashard/datashard_split_dst.cpp b/ydb/core/tx/datashard/datashard_split_dst.cpp index 9cb70cdbd74..24e741dd7b9 100644 --- a/ydb/core/tx/datashard/datashard_split_dst.cpp +++ b/ydb/core/tx/datashard/datashard_split_dst.cpp @@ -361,7 +361,7 @@ public: auto itUserTables = userTables.find(msg->PathId.LocalPathId); Y_VERIFY(itUserTables != userTables.end()); TUserTable::TCPtr tableInfo = itUserTables->second; - TConstArrayRef<NScheme::TTypeId> keyColumnTypes = tableInfo->KeyColumnTypes; + TConstArrayRef<NScheme::TTypeInfo> keyColumnTypes = tableInfo->KeyColumnTypes; auto* replTable = Self->EnsureReplicatedTable(msg->PathId); Y_VERIFY(replTable); diff --git a/ydb/core/tx/datashard/datashard_user_table.cpp b/ydb/core/tx/datashard/datashard_user_table.cpp index b7112cd6f9a..c78cac2c911 100644 --- a/ydb/core/tx/datashard/datashard_user_table.cpp +++ b/ydb/core/tx/datashard/datashard_user_table.cpp @@ -1,6 +1,7 @@ #include "datashard_user_table.h" #include <ydb/core/base/path.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/tablet_flat/flat_cxx_database.h> #include <ydb/core/tablet_flat/tablet_flat_executed.h> #include <ydb/core/tablet_flat/tablet_flat_executor.h> @@ -218,7 +219,9 @@ void TUserTable::ParseProto(const NKikimrSchemeOp::TTableDescription& descr) for (const auto& col : descr.GetColumns()) { TUserColumn& column = Columns[col.GetId()]; if (column.Name.empty()) { - column = TUserColumn(col.GetTypeId(), col.GetName()); + auto typeInfo = NScheme::TypeInfoFromProtoColumnType(col.GetTypeId(), + col.HasTypeInfo() ? &col.GetTypeInfo() : nullptr); + column = TUserColumn(typeInfo, col.GetName()); } column.Family = col.GetFamily(); column.NotNull = col.GetNotNull(); @@ -289,7 +292,7 @@ void TUserTable::CheckSpecialColumns() { const ui32 colId = xpair.first; const auto &column = xpair.second; - if (column.IsKey || column.Type != NScheme::NTypeIds::Uint64) + if (column.IsKey || column.Type.GetTypeId() != NScheme::NTypeIds::Uint64) continue; if (column.Name == "__tablet") @@ -331,7 +334,11 @@ void TUserTable::AlterSchema() { auto descr = schema.AddColumns(); descr->SetName(column.Name); descr->SetId(col.first); - descr->SetTypeId(column.Type); + auto protoType = NScheme::ProtoColumnTypeFromTypeInfo(column.Type); + descr->SetTypeId(protoType.TypeId); + if (protoType.TypeInfo) { + *descr->MutableTypeInfo() = *protoType.TypeInfo; + } descr->SetFamily(column.Family); descr->SetNotNull(column.NotNull); } @@ -390,7 +397,9 @@ void TUserTable::DoApplyCreate( ui32 columnId = col.first; const TUserColumn& column = col.second; - alter.AddColumn(tid, column.Name, columnId, column.Type, column.NotNull); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.Type); + ui32 pgTypeId = columnType.TypeInfo ? columnType.TypeInfo->GetPgTypeId() : 0; + alter.AddPgColumn(tid, column.Name, columnId, columnType.TypeId, pgTypeId, column.NotNull); alter.AddColumnToFamily(tid, columnId, column.Family); } @@ -492,7 +501,9 @@ void TUserTable::ApplyAlter( if (!oldTable.Columns.contains(colId)) { for (ui32 tid : tids) { - alter.AddColumn(tid, column.Name, colId, column.Type, column.NotNull); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.Type); + ui32 pgTypeId = columnType.TypeInfo ? columnType.TypeInfo->GetPgTypeId() : 0; + alter.AddPgColumn(tid, column.Name, colId, columnType.TypeId, pgTypeId, column.NotNull); } } diff --git a/ydb/core/tx/datashard/datashard_user_table.h b/ydb/core/tx/datashard/datashard_user_table.h index f3b6f8112c4..3ffd499d372 100644 --- a/ydb/core/tx/datashard/datashard_user_table.h +++ b/ydb/core/tx/datashard/datashard_user_table.h @@ -232,13 +232,13 @@ struct TUserTable : public TThrRefBase { }; struct TUserColumn { - NScheme::TTypeId Type; + NScheme::TTypeInfo Type; TString Name; bool IsKey; ui32 Family = 0; bool NotNull = false; - TUserColumn(NScheme::TTypeId type, TString name, bool isKey = false) + TUserColumn(NScheme::TTypeInfo type, TString name, bool isKey = false) : Type(type) , Name(name) , IsKey(isKey) @@ -351,7 +351,7 @@ struct TUserTable : public TThrRefBase { TMap<ui32, TStorageRoom::TPtr> Rooms; TMap<ui32, TUserFamily> Families; TMap<ui32, TUserColumn> Columns; - TVector<NScheme::TTypeId> KeyColumnTypes; + TVector<NScheme::TTypeInfo> KeyColumnTypes; TVector<ui32> KeyColumnIds; TSerializedTableRange Range; bool IsBackup = false; diff --git a/ydb/core/tx/datashard/datashard_ut_common.cpp b/ydb/core/tx/datashard/datashard_ut_common.cpp index 8954f967bc7..83554e0834c 100644 --- a/ydb/core/tx/datashard/datashard_ut_common.cpp +++ b/ydb/core/tx/datashard/datashard_ut_common.cpp @@ -3,6 +3,7 @@ #include <ydb/core/base/tablet.h> #include <ydb/core/base/tablet_resolver.h> #include <ydb/core/scheme/scheme_types_defs.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/tablet_flat/flat_bio_events.h> #include <ydb/core/tablet_flat/shared_cache_events.h> #include <ydb/core/testlib/basics/appdata.h> @@ -198,7 +199,9 @@ void TTester::RegisterTableInResolver(const TString& schemeText) keyIdx = ki; } } - table.Columns.insert(std::make_pair(c.GetName(), TColumn{c.GetId(), keyIdx, c.GetTypeId(), 0, EColumnTypeConstraint::Nullable})); + auto typeInfo = NScheme::TypeInfoFromProtoColumnType(c.GetTypeId(), + c.HasTypeInfo() ? &c.GetTypeInfo() : nullptr); + table.Columns.insert(std::make_pair(c.GetName(), TColumn{c.GetId(), keyIdx, typeInfo, 0, EColumnTypeConstraint::Nullable})); } DbSchemeResolver.AddTable(table); } diff --git a/ydb/core/tx/datashard/datashard_ut_locks.cpp b/ydb/core/tx/datashard/datashard_ut_locks.cpp index 8f15575378a..b334d4400f7 100644 --- a/ydb/core/tx/datashard/datashard_ut_locks.cpp +++ b/ydb/core/tx/datashard/datashard_ut_locks.cpp @@ -55,7 +55,7 @@ namespace NTest { void InitSchema() { TableInfos[EUserTableId].LocalTid = EUserTableId; TableInfos[EUserTableId].Name = "user____Table"; - TableInfos[EUserTableId].KeyColumnTypes.push_back(NScheme::NTypeIds::Uint32); + TableInfos[EUserTableId].KeyColumnTypes.push_back(NScheme::TTypeInfo(NScheme::NTypeIds::Uint32)); NTable::TAlter delta; diff --git a/ydb/core/tx/datashard/datashard_ut_range_ops.cpp b/ydb/core/tx/datashard/datashard_ut_range_ops.cpp index 681749709bd..053f25b85dc 100644 --- a/ydb/core/tx/datashard/datashard_ut_range_ops.cpp +++ b/ydb/core/tx/datashard/datashard_ut_range_ops.cpp @@ -29,7 +29,7 @@ Y_UNIT_TEST_SUITE(RangeOps) { return TSerializedTableRange(from, to, fromInclusive, toInclusive); } - void CheckRange(TConstArrayRef<NScheme::TTypeId> types, + void CheckRange(TConstArrayRef<NScheme::TTypeInfo> types, const TTableRange& first, const TTableRange& second) { if (first.IsEmptyRange(types)) { Y_ASSERT(second.IsEmptyRange(types)); @@ -56,8 +56,9 @@ Y_UNIT_TEST_SUITE(RangeOps) { auto typeRegistry = MakeHolder<NScheme::TKikimrTypeRegistry>(); typeRegistry->CalculateMetadataEtag(); - const TVector<NKikimr::NScheme::TTypeId> valueType = {NScheme::NTypeIds::Uint64}; - const TVector<NKikimr::NScheme::TTypeId> pairType = {NScheme::NTypeIds::Uint64, NScheme::NTypeIds::Uint64}; + auto typeInfoUi64 = NScheme::TTypeInfo(NScheme::NTypeIds::Uint64); + const TVector<NScheme::TTypeInfo> valueType = {typeInfoUi64}; + const TVector<NScheme::TTypeInfo> pairType = {typeInfoUi64, typeInfoUi64}; const auto emptyRange = MakeRange( MakePoint({20}), true, diff --git a/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp b/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp index 19de40a2b51..dc39356ec0d 100644 --- a/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp +++ b/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp @@ -76,7 +76,7 @@ struct TRowWriter : public NArrow::IRowWriter { }; std::vector<TOwnedCellVec> GetRows( - const TVector<std::pair<TString, NScheme::TTypeId>>& batchSchema, + const TVector<std::pair<TString, NScheme::TTypeInfo>>& batchSchema, const TEvDataShard::TEvReadResult& result) { UNIT_ASSERT(result.ArrowBatch); @@ -94,7 +94,7 @@ std::vector<TOwnedCellVec> GetRows( void CheckRow( const TConstArrayRef<TCell>& row, const TCellVec& gold, - const std::vector<NScheme::TTypeIdOrder>& goldTypes) + const std::vector<NScheme::TTypeInfoOrder>& goldTypes) { UNIT_ASSERT_VALUES_EQUAL(row.size(), gold.size()); for (size_t i: xrange(row.size())) { @@ -107,7 +107,7 @@ template <typename TCellVecType> void CheckRows( const std::vector<TCellVecType>& rows, const std::vector<TCellVec>& gold, - const std::vector<NScheme::TTypeIdOrder>& goldTypes) + const std::vector<NScheme::TTypeInfoOrder>& goldTypes) { UNIT_ASSERT_VALUES_EQUAL(rows.size(), gold.size()); for (size_t i: xrange(rows.size())) { @@ -119,7 +119,7 @@ void CheckResultCellVec( const NKikimrTxDataShard::TEvGetInfoResponse::TUserTable& userTable, const TEvDataShard::TEvReadResult& result, const std::vector<TCellVec>& gold, - const std::vector<NScheme::TTypeIdOrder>& goldTypes, + const std::vector<NScheme::TTypeInfoOrder>& goldTypes, std::vector<NTable::TTag> columns = {}) { Y_UNUSED(userTable); @@ -142,13 +142,13 @@ void CheckResultArrow( const NKikimrTxDataShard::TEvGetInfoResponse::TUserTable& userTable, const TEvDataShard::TEvReadResult& result, const std::vector<TCellVec>& gold, - const std::vector<NScheme::TTypeIdOrder>& goldTypes, + const std::vector<NScheme::TTypeInfoOrder>& goldTypes, std::vector<NTable::TTag> columns = {}) { UNIT_ASSERT(!gold.empty()); UNIT_ASSERT(result.ArrowBatch); - TVector<std::pair<TString, NScheme::TTypeId>> batchSchema; + TVector<std::pair<TString, NScheme::TTypeInfo>> batchSchema; const auto& description = userTable.GetDescription(); if (columns.empty()) { batchSchema.reserve(description.ColumnsSize()); @@ -175,7 +175,7 @@ void CheckResult( const NKikimrTxDataShard::TEvGetInfoResponse::TUserTable& userTable, const TEvDataShard::TEvReadResult& result, const std::vector<TCellVec>& gold, - const std::vector<NScheme::TTypeIdOrder>& goldTypes, + const std::vector<NScheme::TTypeInfoOrder>& goldTypes, std::vector<NTable::TTag> columns = {}) { const auto& record = result.Record; @@ -211,12 +211,12 @@ void CheckResult( const std::vector<std::vector<ui32>>& gold, std::vector<NTable::TTag> columns = {}) { - std::vector<NScheme::TTypeIdOrder> types; + std::vector<NScheme::TTypeInfoOrder> types; if (!gold.empty() && !gold[0].empty()) { types.reserve(gold[0].size()); for (auto i: xrange(gold[0].size())) { Y_UNUSED(i); - types.emplace_back(NScheme::NTypeIds::Uint32); + types.emplace_back(NScheme::TTypeInfo(NScheme::NTypeIds::Uint32)); } } diff --git a/ydb/core/tx/datashard/datashard_ut_snapshot.cpp b/ydb/core/tx/datashard/datashard_ut_snapshot.cpp index b75720bb163..d63d3774c73 100644 --- a/ydb/core/tx/datashard/datashard_ut_snapshot.cpp +++ b/ydb/core/tx/datashard/datashard_ut_snapshot.cpp @@ -23,7 +23,7 @@ namespace { class TTextFormatBuilder : public IBlockBuilder { public: bool Start( - const TVector<std::pair<TString, NScheme::TTypeId>>& columns, + const TVector<std::pair<TString, NScheme::TTypeInfo>>& columns, ui64 maxRowsInBlock, ui64 maxBytesInBlock, TString& err) override @@ -65,7 +65,7 @@ namespace { } private: - TVector<std::pair<TString, NScheme::TTypeId>> Columns; + TVector<std::pair<TString, NScheme::TTypeInfo>> Columns; TString Buffer; }; diff --git a/ydb/core/tx/datashard/erase_rows_condition.cpp b/ydb/core/tx/datashard/erase_rows_condition.cpp index d6960773513..e7b06ce59cf 100644 --- a/ydb/core/tx/datashard/erase_rows_condition.cpp +++ b/ydb/core/tx/datashard/erase_rows_condition.cpp @@ -131,7 +131,8 @@ public: Pos = remapPos.GetOrElse(columnInfo->Pos); Y_VERIFY(Pos < scheme->Tags().size()); - Type = columnInfo->TypeId; + Type = columnInfo->TypeInfo.GetTypeId(); + Y_VERIFY(Type != NScheme::NTypeIds::Pg, "pg types are not supported"); } bool Check(const NTable::TRowState& row) const override { diff --git a/ydb/core/tx/datashard/execute_distributed_erase_tx_unit.cpp b/ydb/core/tx/datashard/execute_distributed_erase_tx_unit.cpp index 0059180e3ae..6e68eabd848 100644 --- a/ydb/core/tx/datashard/execute_distributed_erase_tx_unit.cpp +++ b/ydb/core/tx/datashard/execute_distributed_erase_tx_unit.cpp @@ -119,7 +119,6 @@ public: for (size_t ki : xrange(tableInfo.KeyColumnTypes.size())) { const auto& kt = tableInfo.KeyColumnTypes[ki]; const TCell& cell = keyCells.GetCells()[ki]; - key.emplace_back(TRawTypeValue(cell.AsRef(), kt)); } diff --git a/ydb/core/tx/datashard/export_s3_buffer_raw.cpp b/ydb/core/tx/datashard/export_s3_buffer_raw.cpp index 1ad77ed46d1..f5b0efbd2f6 100644 --- a/ydb/core/tx/datashard/export_s3_buffer_raw.cpp +++ b/ydb/core/tx/datashard/export_s3_buffer_raw.cpp @@ -57,7 +57,7 @@ void TS3BufferRaw::Collect(const NTable::IScan::TRow& row, IOutputStream& out) { continue; } - switch (column.Type) { + switch (column.Type.GetTypeId()) { case NScheme::NTypeIds::Int32: out << cell.AsValue<i32>(); break; @@ -121,6 +121,9 @@ void TS3BufferRaw::Collect(const NTable::IScan::TRow& row, IOutputStream& out) { case NScheme::NTypeIds::JsonDocument: out << '"' << CGIEscapeRet(NBinaryJson::SerializeToJson(cell.AsBuf())) << '"'; break; + case NScheme::NTypeIds::Pg: + // TODO: support pg types + Y_FAIL("Unsupported pg type"); default: Y_FAIL("Unsupported type"); } diff --git a/ydb/core/tx/datashard/import_common.h b/ydb/core/tx/datashard/import_common.h index a7cb38b18e7..7ae51dfbfb2 100644 --- a/ydb/core/tx/datashard/import_common.h +++ b/ydb/core/tx/datashard/import_common.h @@ -81,7 +81,7 @@ public: return ColumnNameIndex.contains(name); } - NScheme::TTypeId GetColumnType(const TString& name) const { + NScheme::TTypeInfo GetColumnType(const TString& name) const { auto it = ColumnNameIndex.find(name); Y_VERIFY(it != ColumnNameIndex.end()); return it->second->second.Type; diff --git a/ydb/core/tx/datashard/import_s3.cpp b/ydb/core/tx/datashard/import_s3.cpp index 975e8a226df..7a592b1b150 100644 --- a/ydb/core/tx/datashard/import_s3.cpp +++ b/ydb/core/tx/datashard/import_s3.cpp @@ -9,6 +9,7 @@ #include <ydb/core/base/appdata.h> #include <ydb/core/protos/flat_scheme_op.pb.h> #include <ydb/core/protos/services.pb.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/tablet/resource_broker.h> #include <ydb/core/wrappers/s3_wrapper.h> #include <ydb/core/wrappers/s3_storage.h> @@ -496,11 +497,13 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader> { return false; } - std::vector<std::pair<i32, ui32>> columnOrderTypes; // {keyOrder, PType} + std::vector<std::pair<i32, NScheme::TTypeInfo>> columnOrderTypes; // {keyOrder, PType} columnOrderTypes.reserve(Scheme.GetColumns().size()); for (const auto& column : Scheme.GetColumns()) { - columnOrderTypes.emplace_back(TableInfo.KeyOrder(column.GetName()), column.GetTypeId()); + auto typeInfo = NScheme::TypeInfoFromProtoColumnType(column.GetTypeId(), + column.HasTypeInfo() ? &column.GetTypeInfo() : nullptr); + columnOrderTypes.emplace_back(TableInfo.KeyOrder(column.GetName()), typeInfo); } TVector<TCell> keys; @@ -596,12 +599,14 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader> { << ": name# " << column.GetName()); } - const NScheme::TTypeId type = TableInfo.GetColumnType(column.GetName()); - if (type != static_cast<NScheme::TTypeId>(column.GetTypeId())) { + const auto typeInfo = TableInfo.GetColumnType(column.GetName()); + auto columnTypeInfo = NScheme::TypeInfoFromProtoColumnType(column.GetTypeId(), + column.HasTypeInfo() ? &column.GetTypeInfo() : nullptr); + if (typeInfo != columnTypeInfo) { return finish(TStringBuilder() << "Scheme mismatch: column type mismatch" << ": name# " << column.GetName() - << ", expected# " << type - << ", got# " << static_cast<NScheme::TTypeId>(column.GetTypeId())); + << ", expected# " << NScheme::TypeName(typeInfo) + << ", got# " << NScheme::TypeName(columnTypeInfo)); } } diff --git a/ydb/core/tx/datashard/range_avl_tree_ut.cpp b/ydb/core/tx/datashard/range_avl_tree_ut.cpp index ae88374addb..012c980cd6e 100644 --- a/ydb/core/tx/datashard/range_avl_tree_ut.cpp +++ b/ydb/core/tx/datashard/range_avl_tree_ut.cpp @@ -9,10 +9,11 @@ namespace NDataShard { namespace { - TVector<NScheme::TTypeId> CreateSchema(size_t n) { - TVector<NScheme::TTypeId> schema; + TVector<NScheme::TTypeInfo> CreateSchema(size_t n) { + TVector<NScheme::TTypeInfo> schema; + auto type = NScheme::TTypeInfo(NScheme::NTypeIds::Uint64); for (size_t i = 0; i < n; ++i) { - schema.emplace_back(NScheme::NTypeIds::Uint64); + schema.emplace_back(type); } return schema; } diff --git a/ydb/core/tx/datashard/range_ops.cpp b/ydb/core/tx/datashard/range_ops.cpp index 8566740c387..fd39536029b 100644 --- a/ydb/core/tx/datashard/range_ops.cpp +++ b/ydb/core/tx/datashard/range_ops.cpp @@ -1,6 +1,6 @@ #include "range_ops.h" -NKikimr::TTableRange NKikimr::Intersect(TConstArrayRef<NScheme::TTypeId> types, const TTableRange& first, const TTableRange& second) +NKikimr::TTableRange NKikimr::Intersect(TConstArrayRef<NScheme::TTypeInfo> types, const TTableRange& first, const TTableRange& second) { // all variants //================= @@ -196,7 +196,7 @@ NKikimr::TTableRange NKikimr::Intersect(TConstArrayRef<NScheme::TTypeId> types, } } -TString NKikimr::DebugPrintRange(TConstArrayRef<NScheme::TTypeId> types, const NKikimr::TTableRange &range, +TString NKikimr::DebugPrintRange(TConstArrayRef<NScheme::TTypeInfo> types, const NKikimr::TTableRange &range, const NScheme::TTypeRegistry& typeRegistry) { if (range.Point) { @@ -211,7 +211,7 @@ TString NKikimr::DebugPrintRange(TConstArrayRef<NScheme::TTypeId> types, const N << (range.InclusiveTo ? "]" : ")"); } -TString NKikimr::DebugPrintRanges(TConstArrayRef<NScheme::TTypeId> types, +TString NKikimr::DebugPrintRanges(TConstArrayRef<NScheme::TTypeInfo> types, const TSmallVec<TSerializedTableRange>& ranges, const NScheme::TTypeRegistry& typeRegistry) { auto out = TStringBuilder(); @@ -223,7 +223,7 @@ TString NKikimr::DebugPrintRanges(TConstArrayRef<NScheme::TTypeId> types, return out; } -TString NKikimr::DebugPrintPoint(TConstArrayRef<NScheme::TTypeId> types, const TConstArrayRef<TCell> &point, const NScheme::TTypeRegistry& typeRegistry) { +TString NKikimr::DebugPrintPoint(TConstArrayRef<NScheme::TTypeInfo> types, const TConstArrayRef<TCell> &point, const NScheme::TTypeRegistry& typeRegistry) { Y_VERIFY(types.size() >= point.size()); TDbTupleRef pointRef(types.data(), point.data(), point.size()); @@ -231,7 +231,7 @@ TString NKikimr::DebugPrintPoint(TConstArrayRef<NScheme::TTypeId> types, const T } TString NKikimr::DebugPrintPartitionInfo(const TKeyDesc::TPartitionInfo& partition, - const TVector<NScheme::TTypeId>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) + const TVector<NScheme::TTypeInfo>& keyTypes, const NScheme::TTypeRegistry& typeRegistry) { TStringBuilder range; if (partition.Range) { diff --git a/ydb/core/tx/datashard/range_ops.h b/ydb/core/tx/datashard/range_ops.h index a7098dd893c..b93ee8a48a8 100644 --- a/ydb/core/tx/datashard/range_ops.h +++ b/ydb/core/tx/datashard/range_ops.h @@ -6,13 +6,13 @@ namespace NKikimr { -TTableRange Intersect(TConstArrayRef<NScheme::TTypeId> types, const TTableRange& first, const TTableRange& second); +TTableRange Intersect(TConstArrayRef<NScheme::TTypeInfo> types, const TTableRange& first, const TTableRange& second); -TString DebugPrintRange(TConstArrayRef<NScheme::TTypeId> types, const TTableRange& range, const NScheme::TTypeRegistry& typeRegistry); -TString DebugPrintRanges(TConstArrayRef<NScheme::TTypeId> types, const TSmallVec<TSerializedTableRange>& ranges, const NScheme::TTypeRegistry& typeRegistry); -TString DebugPrintPoint(TConstArrayRef<NScheme::TTypeId> types, const TConstArrayRef<TCell>& point, const NScheme::TTypeRegistry& typeRegistry); +TString DebugPrintRange(TConstArrayRef<NScheme::TTypeInfo> types, const TTableRange& range, const NScheme::TTypeRegistry& typeRegistry); +TString DebugPrintRanges(TConstArrayRef<NScheme::TTypeInfo> types, const TSmallVec<TSerializedTableRange>& ranges, const NScheme::TTypeRegistry& typeRegistry); +TString DebugPrintPoint(TConstArrayRef<NScheme::TTypeInfo> types, const TConstArrayRef<TCell>& point, const NScheme::TTypeRegistry& typeRegistry); -TString DebugPrintPartitionInfo(const TKeyDesc::TPartitionInfo& partition, const TVector<NScheme::TTypeId>& keyTypes, +TString DebugPrintPartitionInfo(const TKeyDesc::TPartitionInfo& partition, const TVector<NScheme::TTypeInfo>& keyTypes, const NScheme::TTypeRegistry& typeRegistry); } diff --git a/ydb/core/tx/datashard/range_treap_ut.cpp b/ydb/core/tx/datashard/range_treap_ut.cpp index d9b481b8468..aa4717e3e56 100644 --- a/ydb/core/tx/datashard/range_treap_ut.cpp +++ b/ydb/core/tx/datashard/range_treap_ut.cpp @@ -9,10 +9,11 @@ namespace NDataShard { namespace { - TVector<NScheme::TTypeId> CreateSchema(size_t n) { - TVector<NScheme::TTypeId> schema; + TVector<NScheme::TTypeInfo> CreateSchema(size_t n) { + TVector<NScheme::TTypeInfo> schema; + auto type = NScheme::TTypeInfo(NScheme::NTypeIds::Uint64); for (size_t i = 0; i < n; ++i) { - schema.emplace_back(NScheme::NTypeIds::Uint64); + schema.emplace_back(type); } return schema; } diff --git a/ydb/core/tx/datashard/range_tree_base.h b/ydb/core/tx/datashard/range_tree_base.h index 58a66669f05..62191566185 100644 --- a/ydb/core/tx/datashard/range_tree_base.h +++ b/ydb/core/tx/datashard/range_tree_base.h @@ -70,7 +70,7 @@ namespace NDataShard { } public: - void SetKeyTypes(const TVector<NScheme::TTypeId>& keyTypes) { + void SetKeyTypes(const TVector<NScheme::TTypeInfo>& keyTypes) { Y_VERIFY(keyTypes.size() >= KeyTypes.size(), "Number of key columns must not decrease over time"); KeyTypes = keyTypes; } @@ -109,7 +109,7 @@ namespace NDataShard { protected: size_t Size_ = 0; mutable TStats Stats_; - TVector<NScheme::TTypeId> KeyTypes; + TVector<NScheme::TTypeInfo> KeyTypes; }; } // namespace NDataShard diff --git a/ydb/core/tx/datashard/read_table_scan.cpp b/ydb/core/tx/datashard/read_table_scan.cpp index 3eb3ac30482..33b2a65aad1 100644 --- a/ydb/core/tx/datashard/read_table_scan.cpp +++ b/ydb/core/tx/datashard/read_table_scan.cpp @@ -5,6 +5,7 @@ #include <ydb/core/engine/mkql_proto.h> #include <ydb/core/protos/counters_datashard.pb.h> #include <ydb/core/protos/ydb_result_set_old.pb.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/library/binary_json/read.h> #include <ydb/library/dynumber/dynumber.h> @@ -20,7 +21,7 @@ using NTable::EScan; //YdbOld and Ydb.v1 have same value representation template<typename TOutValue> -Y_FORCE_INLINE void AddCell(TOutValue& row, NScheme::TTypeId type, const TCell &cell) +Y_FORCE_INLINE void AddCell(TOutValue& row, NScheme::TTypeInfo type, const TCell &cell) { auto &val = *row.add_items(); @@ -29,7 +30,7 @@ Y_FORCE_INLINE void AddCell(TOutValue& row, NScheme::TTypeId type, const TCell & return; } - switch (type) { + switch (type.GetTypeId()) { case NUdf::TDataType<bool>::Id: val.set_bool_value(cell.AsValue<bool>()); break; @@ -105,6 +106,10 @@ Y_FORCE_INLINE void AddCell(TOutValue& row, NScheme::TTypeId type, const TCell & val.set_text_value(*number); break; } + case NScheme::NTypeIds::Pg: { + val.set_bytes_value(cell.Data(), cell.Size()); + break; + } default: val.set_bytes_value(cell.Data(), cell.Size()); } @@ -117,8 +122,11 @@ public: , ReservedSize(0) , ResultStream(ResultString) { - for (auto &col : request.GetColumns()) - ColTypes.push_back(col.GetTypeId()); + for (auto &col : request.GetColumns()) { + auto typeInfo = NScheme::TypeInfoFromProtoColumnType(col.GetTypeId(), + col.HasTypeInfo() ? &col.GetTypeInfo() : nullptr); + ColTypes.push_back(typeInfo); + } } virtual ~TRowsToResult() = default; @@ -170,7 +178,7 @@ private: protected: TStringOutput ResultStream; - TVector<NScheme::TTypeId> ColTypes; + TVector<NScheme::TTypeInfo> ColTypes; TVector<ui32> RowOffsets; TString ResultCommon; }; @@ -248,15 +256,26 @@ private: for (auto &col : request.GetColumns()) { auto *meta = res.add_columns(); meta->set_name(col.GetName()); - auto id = static_cast<NYql::NProto::TypeIds>(col.GetTypeId()); - if (id == NYql::NProto::Decimal) { - auto decimalType = meta->mutable_type()->mutable_optional_type()->mutable_item()->mutable_decimal_type(); - //TODO: Pass decimal params here - decimalType->set_precision(22); - decimalType->set_scale(9); + + auto typeInfo = NScheme::TypeInfoFromProtoColumnType(col.GetTypeId(), + col.HasTypeInfo() ? &col.GetTypeInfo() : nullptr); + + if (col.GetTypeId() == NScheme::NTypeIds::Pg) { + auto pgType = meta->mutable_type()->mutable_optional_type()->mutable_item() + ->mutable_pg_type(); + pgType->set_oid(NPg::PgTypeIdFromTypeDesc(typeInfo.GetTypeDesc())); } else { - meta->mutable_type()->mutable_optional_type()->mutable_item() + auto id = static_cast<NYql::NProto::TypeIds>(col.GetTypeId()); + if (id == NYql::NProto::Decimal) { + auto decimalType = meta->mutable_type()->mutable_optional_type()->mutable_item() + ->mutable_decimal_type(); + //TODO: Pass decimal params here + decimalType->set_precision(22); + decimalType->set_scale(9); + } else { + meta->mutable_type()->mutable_optional_type()->mutable_item() ->set_type_id(static_cast<Ydb::Type::PrimitiveTypeId>(id)); + } } } res.set_truncated(true); diff --git a/ydb/core/tx/datashard/sys_tables.h b/ydb/core/tx/datashard/sys_tables.h index e9887bd8dbb..ff02286561b 100644 --- a/ydb/core/tx/datashard/sys_tables.h +++ b/ydb/core/tx/datashard/sys_tables.h @@ -10,12 +10,12 @@ struct TSysTables { struct TTableColumnInfo { TString Name; ui32 Id = 0; - ui32 PType = 0; + NScheme::TTypeInfo PType; i32 KeyOrder = -1; TTableColumnInfo() = default; - TTableColumnInfo(TString name, ui32 colId, ui32 type, i32 keyOrder = -1) + TTableColumnInfo(TString name, ui32 colId, NScheme::TTypeInfo type, i32 keyOrder = -1) : Name(name) , Id(colId) , PType(type) @@ -113,12 +113,14 @@ struct TSysTables { return ""; } - static void GetInfo(THashMap<ui32, TTableColumnInfo>& columns, TVector<ui32>& keyTypes, bool v2) { + static void GetInfo(THashMap<ui32, TTableColumnInfo>& columns, TVector<NScheme::TTypeInfo>& keyTypes, bool v2) { /// @warning Generation is uint32. Don't ask me why - auto type = NScheme::TUint64::TypeId; + auto type = NScheme::TTypeInfo(NScheme::TUint64::TypeId); + auto typeUi32 = NScheme::TTypeInfo(NScheme::TUint32::TypeId); + columns[0] = TTableColumnInfo(GetColName(EColumns::LockId), (ui32)EColumns::LockId, type, 0); columns[1] = TTableColumnInfo(GetColName(EColumns::DataShard), (ui32)EColumns::DataShard, type, 1); - columns[2] = TTableColumnInfo(GetColName(EColumns::Generation), (ui32)EColumns::Generation, NScheme::TUint32::TypeId); + columns[2] = TTableColumnInfo(GetColName(EColumns::Generation), (ui32)EColumns::Generation, typeUi32); columns[3] = TTableColumnInfo(GetColName(EColumns::Counter), (ui32)EColumns::Counter, type); keyTypes.push_back(type); diff --git a/ydb/core/tx/datashard/ut_background_compaction/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_background_compaction/CMakeLists.darwin.txt index b104ecc1fcf..b386be8852f 100644 --- a/ydb/core/tx/datashard/ut_background_compaction/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_background_compaction/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_background_compaction PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_background_compaction/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_background_compaction/CMakeLists.linux.txt index b5005b509eb..029e91f0e2e 100644 --- a/ydb/core/tx/datashard/ut_background_compaction/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_background_compaction/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_background_compaction PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_build_index/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_build_index/CMakeLists.darwin.txt index b6be99c3710..f93c32d2c27 100644 --- a/ydb/core/tx/datashard/ut_build_index/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_build_index/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_build_index PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_build_index/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_build_index/CMakeLists.linux.txt index 9ae52ff09bc..9009d8b9366 100644 --- a/ydb/core/tx/datashard/ut_build_index/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_build_index/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_build_index PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_change_collector/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_change_collector/CMakeLists.darwin.txt index 35a98725ccf..8d6ae9f7ac8 100644 --- a/ydb/core/tx/datashard/ut_change_collector/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_change_collector/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_change_collector PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_change_collector/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_change_collector/CMakeLists.linux.txt index 7edc6b3b6e2..fdde2d95960 100644 --- a/ydb/core/tx/datashard/ut_change_collector/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_change_collector/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_change_collector PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_change_exchange/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_change_exchange/CMakeLists.darwin.txt index 2575fb01829..fc4f7e8fb9c 100644 --- a/ydb/core/tx/datashard/ut_change_exchange/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_change_exchange/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_change_exchange PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_change_exchange/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_change_exchange/CMakeLists.linux.txt index 080f09525d2..ebcee72d391 100644 --- a/ydb/core/tx/datashard/ut_change_exchange/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_change_exchange/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_change_exchange PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_compaction/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_compaction/CMakeLists.darwin.txt index edb9f97cb29..f8d4b47d5f3 100644 --- a/ydb/core/tx/datashard/ut_compaction/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_compaction/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_compaction PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_compaction/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_compaction/CMakeLists.linux.txt index f3f2e18d4df..7c0dfdea781 100644 --- a/ydb/core/tx/datashard/ut_compaction/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_compaction/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_compaction PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_erase_rows/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_erase_rows/CMakeLists.darwin.txt index fb8a69abdbc..038bf6b917b 100644 --- a/ydb/core/tx/datashard/ut_erase_rows/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_erase_rows/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_erase_rows PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_erase_rows/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_erase_rows/CMakeLists.linux.txt index c1a7bf2ffa5..0df88745a34 100644 --- a/ydb/core/tx/datashard/ut_erase_rows/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_erase_rows/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_erase_rows PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_followers/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_followers/CMakeLists.darwin.txt index 6023c95cbff..da51f3fc236 100644 --- a/ydb/core/tx/datashard/ut_followers/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_followers/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_followers PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_followers/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_followers/CMakeLists.linux.txt index 17960aacb4e..10449720728 100644 --- a/ydb/core/tx/datashard/ut_followers/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_followers/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_followers PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_init/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_init/CMakeLists.darwin.txt index 92caa4797cc..381cd4c7af5 100644 --- a/ydb/core/tx/datashard/ut_init/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_init/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_init PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_init/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_init/CMakeLists.linux.txt index b92c03f3e33..629d5296e31 100644 --- a/ydb/core/tx/datashard/ut_init/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_init/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_init PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_keys/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_keys/CMakeLists.darwin.txt index c0ce3408aa5..1d407b2e0a0 100644 --- a/ydb/core/tx/datashard/ut_keys/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_keys/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_keys PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_keys/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_keys/CMakeLists.linux.txt index ed082308602..57605524693 100644 --- a/ydb/core/tx/datashard/ut_keys/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_keys/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_keys PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_kqp/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_kqp/CMakeLists.darwin.txt index 9bd12cd511f..1131f0a23c3 100644 --- a/ydb/core/tx/datashard/ut_kqp/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_kqp/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_kqp PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_kqp/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_kqp/CMakeLists.linux.txt index 1bb08498b77..e3d62e8852b 100644 --- a/ydb/core/tx/datashard/ut_kqp/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_kqp/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_kqp PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_kqp_errors/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_kqp_errors/CMakeLists.darwin.txt index 29cb81173ad..b54fcb8d461 100644 --- a/ydb/core/tx/datashard/ut_kqp_errors/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_kqp_errors/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_kqp_errors PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_kqp_errors/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_kqp_errors/CMakeLists.linux.txt index 575e718b784..3ecd1fe6adb 100644 --- a/ydb/core/tx/datashard/ut_kqp_errors/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_kqp_errors/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_kqp_errors PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_kqp_scan/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_kqp_scan/CMakeLists.darwin.txt index 4d8841b86b7..f158976859d 100644 --- a/ydb/core/tx/datashard/ut_kqp_scan/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_kqp_scan/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_kqp_scan PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_kqp_scan/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_kqp_scan/CMakeLists.linux.txt index 765928910a2..5f104aac3cb 100644 --- a/ydb/core/tx/datashard/ut_kqp_scan/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_kqp_scan/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_kqp_scan PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_locks/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_locks/CMakeLists.darwin.txt index 17033fa45e8..a4f0177cb78 100644 --- a/ydb/core/tx/datashard/ut_locks/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_locks/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_locks PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_locks/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_locks/CMakeLists.linux.txt index 7ae79cc1a90..28630f9a16b 100644 --- a/ydb/core/tx/datashard/ut_locks/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_locks/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_locks PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_minikql/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_minikql/CMakeLists.darwin.txt index 0dac58cfd59..67298af265e 100644 --- a/ydb/core/tx/datashard/ut_minikql/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_minikql/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_minikql PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_minikql/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_minikql/CMakeLists.linux.txt index cf7fb8a1ac3..6f65f05c353 100644 --- a/ydb/core/tx/datashard/ut_minikql/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_minikql/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_minikql PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_minstep/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_minstep/CMakeLists.darwin.txt index 7d72df2143c..8a450055ae0 100644 --- a/ydb/core/tx/datashard/ut_minstep/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_minstep/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_minstep PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_minstep/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_minstep/CMakeLists.linux.txt index 2d059a7cc0e..6664ad7149e 100644 --- a/ydb/core/tx/datashard/ut_minstep/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_minstep/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_minstep PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_order/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_order/CMakeLists.darwin.txt index 67dfe0c599c..15fa08f39ba 100644 --- a/ydb/core/tx/datashard/ut_order/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_order/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_order PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_order/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_order/CMakeLists.linux.txt index 113f8e99af2..0ffaf5df0fa 100644 --- a/ydb/core/tx/datashard/ut_order/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_order/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_order PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.darwin.txt index b9e791805f9..6c024b31fbe 100644 --- a/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_range_avl_tree PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.linux.txt index 5b681c6d4fc..2da2776d944 100644 --- a/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_range_avl_tree/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_range_avl_tree PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_range_ops/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_range_ops/CMakeLists.darwin.txt index 74a1e43f6cb..7db2547942c 100644 --- a/ydb/core/tx/datashard/ut_range_ops/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_range_ops/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_range_ops PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_range_ops/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_range_ops/CMakeLists.linux.txt index 000000f3ff0..e5289e40580 100644 --- a/ydb/core/tx/datashard/ut_range_ops/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_range_ops/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_range_ops PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_range_treap/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_range_treap/CMakeLists.darwin.txt index cb9b857279f..d26018dac2f 100644 --- a/ydb/core/tx/datashard/ut_range_treap/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_range_treap/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_range_treap PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_range_treap/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_range_treap/CMakeLists.linux.txt index 50c069dfa39..0f19a2ab9dc 100644 --- a/ydb/core/tx/datashard/ut_range_treap/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_range_treap/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_range_treap PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_read_iterator/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_read_iterator/CMakeLists.darwin.txt index 4b5bdd56590..fd9b6038695 100644 --- a/ydb/core/tx/datashard/ut_read_iterator/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_read_iterator/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_read_iterator PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_read_iterator/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_read_iterator/CMakeLists.linux.txt index 3c691ff53e1..8d626fb960b 100644 --- a/ydb/core/tx/datashard/ut_read_iterator/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_read_iterator/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_read_iterator PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_read_table/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_read_table/CMakeLists.darwin.txt index 466b28d9660..bef1935e6a2 100644 --- a/ydb/core/tx/datashard/ut_read_table/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_read_table/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_read_table PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_read_table/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_read_table/CMakeLists.linux.txt index a077aaaa1ac..1a648658c6f 100644 --- a/ydb/core/tx/datashard/ut_read_table/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_read_table/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_read_table PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_reassign/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_reassign/CMakeLists.darwin.txt index cd0f71d633d..f1abbc8858b 100644 --- a/ydb/core/tx/datashard/ut_reassign/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_reassign/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_reassign PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_reassign/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_reassign/CMakeLists.linux.txt index 3e2b02d7739..454fadd175d 100644 --- a/ydb/core/tx/datashard/ut_reassign/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_reassign/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_reassign PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_replication/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_replication/CMakeLists.darwin.txt index 9a3069b16e2..5b8312a5501 100644 --- a/ydb/core/tx/datashard/ut_replication/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_replication/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_replication PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_replication/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_replication/CMakeLists.linux.txt index 7363d477d11..c49b07f1197 100644 --- a/ydb/core/tx/datashard/ut_replication/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_replication/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_replication PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_rs/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_rs/CMakeLists.darwin.txt index 51235842961..b3bc6b645ce 100644 --- a/ydb/core/tx/datashard/ut_rs/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_rs/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_rs PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_rs/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_rs/CMakeLists.linux.txt index 85271f600f5..fa3ae763dbb 100644 --- a/ydb/core/tx/datashard/ut_rs/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_rs/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_rs PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_snapshot/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_snapshot/CMakeLists.darwin.txt index 13574753b6f..f4b25a7759d 100644 --- a/ydb/core/tx/datashard/ut_snapshot/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_snapshot/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_snapshot PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_snapshot/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_snapshot/CMakeLists.linux.txt index 8591db0cba4..3687aad13e0 100644 --- a/ydb/core/tx/datashard/ut_snapshot/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_snapshot/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_snapshot PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_stats/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_stats/CMakeLists.darwin.txt index 247bbc51beb..ea7263a71e8 100644 --- a/ydb/core/tx/datashard/ut_stats/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_stats/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_stats PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_stats/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_stats/CMakeLists.linux.txt index c8bc90d9258..8240a451a19 100644 --- a/ydb/core/tx/datashard/ut_stats/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_stats/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_stats PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_testload/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_testload/CMakeLists.darwin.txt index 5e53efa7975..b81f59a7042 100644 --- a/ydb/core/tx/datashard/ut_testload/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_testload/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_testload PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_testload/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_testload/CMakeLists.linux.txt index d42847ee463..47cde6912cd 100644 --- a/ydb/core/tx/datashard/ut_testload/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_testload/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_testload PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_upload_rows/CMakeLists.darwin.txt b/ydb/core/tx/datashard/ut_upload_rows/CMakeLists.darwin.txt index 5b64904b2dd..f266426caa5 100644 --- a/ydb/core/tx/datashard/ut_upload_rows/CMakeLists.darwin.txt +++ b/ydb/core/tx/datashard/ut_upload_rows/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_upload_rows PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/datashard/ut_upload_rows/CMakeLists.linux.txt b/ydb/core/tx/datashard/ut_upload_rows/CMakeLists.linux.txt index 581b4a31611..14bf513ff14 100644 --- a/ydb/core/tx/datashard/ut_upload_rows/CMakeLists.linux.txt +++ b/ydb/core/tx/datashard/ut_upload_rows/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-datashard-ut_upload_rows PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy public-lib-yson_value diff --git a/ydb/core/tx/long_tx_service/ut/CMakeLists.darwin.txt b/ydb/core/tx/long_tx_service/ut/CMakeLists.darwin.txt index 5e64a14d4cb..f877946598a 100644 --- a/ydb/core/tx/long_tx_service/ut/CMakeLists.darwin.txt +++ b/ydb/core/tx/long_tx_service/ut/CMakeLists.darwin.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-core-tx-long_tx_service-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main core-tx-long_tx_service - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-tx-long_tx_service-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/tx/long_tx_service/ut/CMakeLists.linux.txt b/ydb/core/tx/long_tx_service/ut/CMakeLists.linux.txt index e085e604a31..7750816d62e 100644 --- a/ydb/core/tx/long_tx_service/ut/CMakeLists.linux.txt +++ b/ydb/core/tx/long_tx_service/ut/CMakeLists.linux.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-tx-long_tx_service-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main core-tx-long_tx_service - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-tx-long_tx_service-ut PRIVATE -ldl diff --git a/ydb/core/tx/replication/ydb_proxy/ut/CMakeLists.darwin.txt b/ydb/core/tx/replication/ydb_proxy/ut/CMakeLists.darwin.txt index b0c45e4dffb..bacc77648a2 100644 --- a/ydb/core/tx/replication/ydb_proxy/ut/CMakeLists.darwin.txt +++ b/ydb/core/tx/replication/ydb_proxy/ut/CMakeLists.darwin.txt @@ -21,7 +21,7 @@ target_link_libraries(ydb-core-tx-replication-ydb_proxy-ut PUBLIC cpp-testing-unittest_main tx-replication-ydb_proxy cpp-testing-unittest - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-tx-replication-ydb_proxy-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/tx/replication/ydb_proxy/ut/CMakeLists.linux.txt b/ydb/core/tx/replication/ydb_proxy/ut/CMakeLists.linux.txt index 60b2eed84eb..58000fe54be 100644 --- a/ydb/core/tx/replication/ydb_proxy/ut/CMakeLists.linux.txt +++ b/ydb/core/tx/replication/ydb_proxy/ut/CMakeLists.linux.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-replication-ydb_proxy-ut PUBLIC cpp-testing-unittest_main tx-replication-ydb_proxy cpp-testing-unittest - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-tx-replication-ydb_proxy-ut PRIVATE -ldl diff --git a/ydb/core/tx/scheme_board/cache.cpp b/ydb/core/tx/scheme_board/cache.cpp index 5eec5b9e11d..f862770500a 100644 --- a/ydb/core/tx/scheme_board/cache.cpp +++ b/ydb/core/tx/scheme_board/cache.cpp @@ -14,6 +14,7 @@ #include <ydb/core/protos/flat_tx_scheme.pb.h> #include <ydb/core/protos/services.pb.h> #include <ydb/core/scheme/scheme_tabledefs.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/sys_view/common/schema.h> #include <ydb/core/tx/schemeshard/schemeshard_types.h> #include <ydb/core/util/yverify_stream.h> @@ -744,8 +745,8 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { auto& column = Columns[columnDesc.GetId()]; column.Id = columnDesc.GetId(); column.Name = columnDesc.GetName(); - column.PType = columnDesc.GetTypeId(); - + column.PType = NScheme::TypeInfoFromProtoColumnType(columnDesc.GetTypeId(), + columnDesc.HasTypeInfo() ? &columnDesc.GetTypeInfo() : nullptr); if (columnDesc.GetNotNull()) { NotNullColumns.insert(columnDesc.GetName()); } @@ -805,7 +806,8 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { auto& column = Columns[columnDesc.GetId()]; column.Id = columnDesc.GetId(); column.Name = columnDesc.GetName(); - column.PType = columnDesc.GetTypeId(); + column.PType = NScheme::TypeInfoFromProtoColumnType(columnDesc.GetTypeId(), + columnDesc.HasTypeInfo() ? &columnDesc.GetTypeInfo() : nullptr); nameToId[column.Name] = column.Id; } @@ -1130,7 +1132,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { columns.BeginObject() .WriteKey("Id").WriteULongLong(column.Id) .WriteKey("Name").WriteString(column.Name) - .WriteKey("Type").WriteULongLong(column.PType) + .WriteKey("Type").WriteULongLong(column.PType.GetTypeId()) // TODO: support pg types .WriteKey("KeyOrder").WriteInt(column.KeyOrder) .EndObject(); } @@ -1313,9 +1315,9 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { PathId = TPathId(TSysTables::SysSchemeShard, v2 ? TSysTables::SysTableLocks2 : TSysTables::SysTableLocks); Path = v2 ? "/sys/locks2" : "/sys/locks"; - TVector<ui32> keyColumnTypes; + TVector<NScheme::TTypeInfo> keyColumnTypes; TSysTables::TLocksTable::GetInfo(Columns, keyColumnTypes, v2); - for (ui32 type : keyColumnTypes) { + for (auto type : keyColumnTypes) { KeyColumnTypes.push_back(type); } @@ -1728,7 +1730,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } bool CheckColumns(TResolveContext* context, TResolve::TEntry& entry, - const TVector<NScheme::TTypeId>& keyColumnTypes, + const TVector<NScheme::TTypeInfo>& keyColumnTypes, const THashMap<ui32, TSysTables::TTableColumnInfo>& columns) const { TKeyDesc& keyDesc = *entry.KeyDescription; @@ -1765,7 +1767,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { entry.Status = TResolve::EStatus::TypeCheckError; keyDesc.Status = TKeyDesc::EStatus::TypeCheckFailed; keyDesc.ColumnInfos.push_back({ - columnOp.Column, 0, 0, TKeyDesc::EStatus::NotExists + columnOp.Column, NScheme::TTypeInfo(), 0, TKeyDesc::EStatus::NotExists }); continue; } @@ -1963,7 +1965,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { // table specific THashMap<ui32, TSysTables::TTableColumnInfo> Columns; - TVector<NScheme::TTypeId> KeyColumnTypes; + TVector<NScheme::TTypeInfo> KeyColumnTypes; THashSet<TString> NotNullColumns; TVector<NKikimrSchemeOp::TIndexDescription> Indexes; TVector<NKikimrSchemeOp::TCdcStreamDescription> CdcStreams; diff --git a/ydb/core/tx/scheme_board/cache_ut.cpp b/ydb/core/tx/scheme_board/cache_ut.cpp index 67bc8b04b23..3b92d1a922a 100644 --- a/ydb/core/tx/scheme_board/cache_ut.cpp +++ b/ydb/core/tx/scheme_board/cache_ut.cpp @@ -428,7 +428,7 @@ TResolve::TEntry TCacheTest::TestResolve(const TTableId& tableId, TResolve::ESta tableId, TTableRange({}), TKeyDesc::ERowOperation::Unknown, - TVector<NScheme::TTypeId>(), TVector<TKeyDesc::TColumnOp>() + TVector<NScheme::TTypeInfo>(), TVector<TKeyDesc::TColumnOp>() ); request->ResultSet.emplace_back(std::move(keyDesc)); diff --git a/ydb/core/tx/scheme_board/ut_cache/CMakeLists.darwin.txt b/ydb/core/tx/scheme_board/ut_cache/CMakeLists.darwin.txt index cdf6d444b22..265960eda82 100644 --- a/ydb/core/tx/scheme_board/ut_cache/CMakeLists.darwin.txt +++ b/ydb/core/tx/scheme_board/ut_cache/CMakeLists.darwin.txt @@ -21,7 +21,7 @@ target_link_libraries(ydb-core-tx-scheme_board-ut_cache PUBLIC cpp-testing-unittest_main core-tx-scheme_board cpp-testing-unittest - ydb-core-testlib + core-testlib-default core-tx-schemeshard tx-schemeshard-ut_helpers ) diff --git a/ydb/core/tx/scheme_board/ut_cache/CMakeLists.linux.txt b/ydb/core/tx/scheme_board/ut_cache/CMakeLists.linux.txt index 1253a08e080..7396d937be2 100644 --- a/ydb/core/tx/scheme_board/ut_cache/CMakeLists.linux.txt +++ b/ydb/core/tx/scheme_board/ut_cache/CMakeLists.linux.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-scheme_board-ut_cache PUBLIC cpp-testing-unittest_main core-tx-scheme_board cpp-testing-unittest - ydb-core-testlib + core-testlib-default core-tx-schemeshard tx-schemeshard-ut_helpers ) diff --git a/ydb/core/tx/scheme_board/ut_monitoring/CMakeLists.darwin.txt b/ydb/core/tx/scheme_board/ut_monitoring/CMakeLists.darwin.txt index 47cc57f6e6c..41461692b2c 100644 --- a/ydb/core/tx/scheme_board/ut_monitoring/CMakeLists.darwin.txt +++ b/ydb/core/tx/scheme_board/ut_monitoring/CMakeLists.darwin.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-tx-scheme_board-ut_monitoring PUBLIC core-tx-scheme_board cpp-actors-core cpp-testing-unittest - core-testlib-basics + testlib-basics-default ) target_link_options(ydb-core-tx-scheme_board-ut_monitoring PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/tx/scheme_board/ut_monitoring/CMakeLists.linux.txt b/ydb/core/tx/scheme_board/ut_monitoring/CMakeLists.linux.txt index b74296f1d28..ad52c42a4d6 100644 --- a/ydb/core/tx/scheme_board/ut_monitoring/CMakeLists.linux.txt +++ b/ydb/core/tx/scheme_board/ut_monitoring/CMakeLists.linux.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-scheme_board-ut_monitoring PUBLIC core-tx-scheme_board cpp-actors-core cpp-testing-unittest - core-testlib-basics + testlib-basics-default ) target_link_options(ydb-core-tx-scheme_board-ut_monitoring PRIVATE -ldl diff --git a/ydb/core/tx/scheme_board/ut_populator/CMakeLists.darwin.txt b/ydb/core/tx/scheme_board/ut_populator/CMakeLists.darwin.txt index a98d48e6c2d..56348f4ed28 100644 --- a/ydb/core/tx/scheme_board/ut_populator/CMakeLists.darwin.txt +++ b/ydb/core/tx/scheme_board/ut_populator/CMakeLists.darwin.txt @@ -21,7 +21,7 @@ target_link_libraries(ydb-core-tx-scheme_board-ut_populator PUBLIC cpp-testing-unittest_main core-tx-scheme_board cpp-testing-unittest - ydb-core-testlib + core-testlib-default core-tx-schemeshard tx-schemeshard-ut_helpers core-tx-tx_allocator diff --git a/ydb/core/tx/scheme_board/ut_populator/CMakeLists.linux.txt b/ydb/core/tx/scheme_board/ut_populator/CMakeLists.linux.txt index eaac0ff6808..b53993b76a8 100644 --- a/ydb/core/tx/scheme_board/ut_populator/CMakeLists.linux.txt +++ b/ydb/core/tx/scheme_board/ut_populator/CMakeLists.linux.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-scheme_board-ut_populator PUBLIC cpp-testing-unittest_main core-tx-scheme_board cpp-testing-unittest - ydb-core-testlib + core-testlib-default core-tx-schemeshard tx-schemeshard-ut_helpers core-tx-tx_allocator diff --git a/ydb/core/tx/scheme_board/ut_replica/CMakeLists.darwin.txt b/ydb/core/tx/scheme_board/ut_replica/CMakeLists.darwin.txt index 0b7fc0d976d..9998d46b575 100644 --- a/ydb/core/tx/scheme_board/ut_replica/CMakeLists.darwin.txt +++ b/ydb/core/tx/scheme_board/ut_replica/CMakeLists.darwin.txt @@ -21,7 +21,7 @@ target_link_libraries(ydb-core-tx-scheme_board-ut_replica PUBLIC cpp-testing-unittest_main core-tx-scheme_board cpp-testing-unittest - core-testlib-basics + testlib-basics-default ) target_link_options(ydb-core-tx-scheme_board-ut_replica PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/tx/scheme_board/ut_replica/CMakeLists.linux.txt b/ydb/core/tx/scheme_board/ut_replica/CMakeLists.linux.txt index 1e0b47b40b1..b6ad9ee7123 100644 --- a/ydb/core/tx/scheme_board/ut_replica/CMakeLists.linux.txt +++ b/ydb/core/tx/scheme_board/ut_replica/CMakeLists.linux.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-scheme_board-ut_replica PUBLIC cpp-testing-unittest_main core-tx-scheme_board cpp-testing-unittest - core-testlib-basics + testlib-basics-default ) target_link_options(ydb-core-tx-scheme_board-ut_replica PRIVATE -ldl diff --git a/ydb/core/tx/scheme_board/ut_subscriber/CMakeLists.darwin.txt b/ydb/core/tx/scheme_board/ut_subscriber/CMakeLists.darwin.txt index 0c183334b78..24b5ef04efc 100644 --- a/ydb/core/tx/scheme_board/ut_subscriber/CMakeLists.darwin.txt +++ b/ydb/core/tx/scheme_board/ut_subscriber/CMakeLists.darwin.txt @@ -22,8 +22,7 @@ target_link_libraries(ydb-core-tx-scheme_board-ut_subscriber PUBLIC core-tx-scheme_board cpp-actors-interconnect cpp-testing-unittest - core-testlib-basics - yql-sql-pg_dummy + testlib-basics-default ) target_link_options(ydb-core-tx-scheme_board-ut_subscriber PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/tx/scheme_board/ut_subscriber/CMakeLists.linux.txt b/ydb/core/tx/scheme_board/ut_subscriber/CMakeLists.linux.txt index bb9271a895e..7b3531c00a8 100644 --- a/ydb/core/tx/scheme_board/ut_subscriber/CMakeLists.linux.txt +++ b/ydb/core/tx/scheme_board/ut_subscriber/CMakeLists.linux.txt @@ -24,8 +24,7 @@ target_link_libraries(ydb-core-tx-scheme_board-ut_subscriber PUBLIC core-tx-scheme_board cpp-actors-interconnect cpp-testing-unittest - core-testlib-basics - yql-sql-pg_dummy + testlib-basics-default ) target_link_options(ydb-core-tx-scheme_board-ut_subscriber PRIVATE -ldl diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 2f478464b36..94517ae74ea 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -1,5 +1,6 @@ #include "schemeshard_impl.h" +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/tablet/tablet_exception.h> #include <ydb/core/tablet_flat/flat_cxx_database.h> #include <ydb/core/util/pb.h> @@ -385,7 +386,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { return true; } - typedef std::tuple<TPathId, ui32, TString, NScheme::TTypeId, ui32, ui64, ui64, ui32, ETableColumnDefaultKind, TString, bool> TColumnRec; + typedef std::tuple<TPathId, ui32, TString, NScheme::TTypeInfo, ui32, ui64, ui64, ui32, ETableColumnDefaultKind, TString, bool> TColumnRec; typedef TDeque<TColumnRec> TColumnRows; bool LoadColumns(NIceDb::TNiceDb& db, TColumnRows& columnRows) const { @@ -401,6 +402,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { ui32 colId = rowSet.GetValue<Schema::Columns::ColId>(); TString colName = rowSet.GetValue<Schema::Columns::ColName>(); NScheme::TTypeId typeId = (NScheme::TTypeId)rowSet.GetValue<Schema::Columns::ColType>(); + TString typeData = rowSet.GetValueOrDefault<Schema::Columns::ColTypeData>(""); ui32 keyOrder = rowSet.GetValue<Schema::Columns::ColKeyOrder>(); ui64 createVersion = rowSet.GetValueOrDefault<Schema::Columns::CreateVersion>(0); ui64 deleteVersion = rowSet.GetValueOrDefault<Schema::Columns::DeleteVersion>(-1); @@ -409,8 +411,17 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { auto defaultValue = rowSet.GetValue<Schema::Columns::DefaultValue>(); auto notNull = rowSet.GetValueOrDefault<Schema::Columns::NotNull>(false); + NScheme::TTypeInfo typeInfo; + if (typeData) { + NKikimrProto::TTypeInfo protoType; + Y_VERIFY(ParseFromStringNoSizeLimit(protoType, typeData)); + typeInfo = NScheme::TypeInfoFromProtoColumnType(typeId, &protoType); + } else { + typeInfo = NScheme::TTypeInfo(typeId); + } + columnRows.emplace_back(pathId, colId, - colName, typeId, keyOrder, createVersion, deleteVersion, + colName, typeInfo, keyOrder, createVersion, deleteVersion, family, defaultKind, defaultValue, notNull); if (!rowSet.Next()) { @@ -433,6 +444,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { ui32 colId = rowSet.GetValue<Schema::MigratedColumns::ColId>(); TString colName = rowSet.GetValue<Schema::MigratedColumns::ColName>(); NScheme::TTypeId typeId = (NScheme::TTypeId)rowSet.GetValue<Schema::MigratedColumns::ColType>(); + TString typeData = rowSet.GetValueOrDefault<Schema::MigratedColumns::ColTypeData>(""); ui32 keyOrder = rowSet.GetValue<Schema::MigratedColumns::ColKeyOrder>(); ui64 createVersion = rowSet.GetValueOrDefault<Schema::MigratedColumns::CreateVersion>(0); ui64 deleteVersion = rowSet.GetValueOrDefault<Schema::MigratedColumns::DeleteVersion>(-1); @@ -441,8 +453,17 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { auto defaultValue = rowSet.GetValue<Schema::MigratedColumns::DefaultValue>(); auto notNull = rowSet.GetValueOrDefault<Schema::MigratedColumns::NotNull>(false); + NScheme::TTypeInfo typeInfo; + if (typeData) { + NKikimrProto::TTypeInfo protoType; + Y_VERIFY(ParseFromStringNoSizeLimit(protoType, typeData)); + typeInfo = NScheme::TypeInfoFromProtoColumnType(typeId, &protoType); + } else { + typeInfo = NScheme::TTypeInfo(typeId); + } + columnRows.emplace_back(pathId, colId, - colName, typeId, keyOrder, createVersion, deleteVersion, + colName, typeInfo, keyOrder, createVersion, deleteVersion, family, defaultKind, defaultValue, notNull); if (!rowSet.Next()) { @@ -467,6 +488,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { ui32 colId = rowSet.GetValue<Schema::ColumnAlters::ColId>(); TString colName = rowSet.GetValue<Schema::ColumnAlters::ColName>(); NScheme::TTypeId typeId = (NScheme::TTypeId)rowSet.GetValue<Schema::ColumnAlters::ColType>(); + TString typeData = rowSet.GetValue<Schema::ColumnAlters::ColTypeData>(); ui32 keyOrder = rowSet.GetValue<Schema::ColumnAlters::ColKeyOrder>(); ui64 createVersion = rowSet.GetValueOrDefault<Schema::ColumnAlters::CreateVersion>(0); ui64 deleteVersion = rowSet.GetValueOrDefault<Schema::ColumnAlters::DeleteVersion>(-1); @@ -475,8 +497,17 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { auto defaultValue = rowSet.GetValue<Schema::ColumnAlters::DefaultValue>(); auto notNull = rowSet.GetValueOrDefault<Schema::ColumnAlters::NotNull>(false); + NScheme::TTypeInfo typeInfo; + if (typeData) { + NKikimrProto::TTypeInfo protoType; + Y_VERIFY(ParseFromStringNoSizeLimit(protoType, typeData)); + typeInfo = NScheme::TypeInfoFromProtoColumnType(typeId, &protoType); + } else { + typeInfo = NScheme::TTypeInfo(typeId); + } + columnRows.emplace_back(pathId, colId, - colName, typeId, keyOrder, createVersion, deleteVersion, + colName, typeInfo, keyOrder, createVersion, deleteVersion, family, defaultKind, defaultValue, notNull); if (!rowSet.Next()) { @@ -499,6 +530,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { ui32 colId = rowSet.GetValue<Schema::MigratedColumnAlters::ColId>(); TString colName = rowSet.GetValue<Schema::MigratedColumnAlters::ColName>(); NScheme::TTypeId typeId = (NScheme::TTypeId)rowSet.GetValue<Schema::MigratedColumnAlters::ColType>(); + TString typeData = rowSet.GetValueOrDefault<Schema::MigratedColumnAlters::ColTypeData>(""); ui32 keyOrder = rowSet.GetValue<Schema::MigratedColumnAlters::ColKeyOrder>(); ui64 createVersion = rowSet.GetValueOrDefault<Schema::MigratedColumnAlters::CreateVersion>(0); ui64 deleteVersion = rowSet.GetValueOrDefault<Schema::MigratedColumnAlters::DeleteVersion>(-1); @@ -507,8 +539,17 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { auto defaultValue = rowSet.GetValue<Schema::MigratedColumnAlters::DefaultValue>(); auto notNull = rowSet.GetValueOrDefault<Schema::MigratedColumnAlters::NotNull>(false); + NScheme::TTypeInfo typeInfo; + if (typeData) { + NKikimrProto::TTypeInfo protoType; + Y_VERIFY(ParseFromStringNoSizeLimit(protoType, typeData)); + typeInfo = NScheme::TypeInfoFromProtoColumnType(typeId, &protoType); + } else { + typeInfo = NScheme::TTypeInfo(typeId); + } + columnRows.emplace_back(pathId, colId, - colName, typeId, keyOrder, createVersion, deleteVersion, + colName, typeInfo, keyOrder, createVersion, deleteVersion, family, defaultKind, defaultValue, notNull); if (!rowSet.Next()) { @@ -1851,7 +1892,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { TPathId pathId = std::get<0>(rec); ui32 colId = std::get<1>(rec); TString colName = std::get<2>(rec); - NScheme::TTypeId typeId = std::get<3>(rec); + NScheme::TTypeInfo typeInfo = std::get<3>(rec); ui32 keyOrder = std::get<4>(rec); ui64 createVersion = std::get<5>(rec); ui64 deleteVersion = std::get<6>(rec); @@ -1869,7 +1910,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { << ", columnId: " << colId << ", NextColId: " << tableInfo->NextColumnId); - TTableInfo::TColumn colInfo(colName, colId, typeId); + TTableInfo::TColumn colInfo(colName, colId, typeInfo); colInfo.KeyOrder = keyOrder; colInfo.CreateVersion = createVersion; colInfo.DeleteVersion = deleteVersion; @@ -1903,7 +1944,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { TPathId pathId = std::get<0>(rec); ui32 colId = std::get<1>(rec); TString colName = std::get<2>(rec); - NScheme::TTypeId typeId = std::get<3>(rec); + NScheme::TTypeInfo typeInfo = std::get<3>(rec); ui32 keyOrder = std::get<4>(rec); ui64 createVersion = std::get<5>(rec); ui64 deleteVersion = std::get<6>(rec); @@ -1923,7 +1964,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { tableInfo->AlterData->NextColumnId = colId + 1; // calc next NextColumnId } - TTableInfo::TColumn colInfo(colName, colId, typeId); + TTableInfo::TColumn colInfo(colName, colId, typeInfo); colInfo.KeyOrder = keyOrder; colInfo.CreateVersion = createVersion; colInfo.DeleteVersion = deleteVersion; diff --git a/ydb/core/tx/schemeshard/schemeshard__monitoring.cpp b/ydb/core/tx/schemeshard/schemeshard__monitoring.cpp index 6b0f1f7eb1d..5b87929a81a 100644 --- a/ydb/core/tx/schemeshard/schemeshard__monitoring.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__monitoring.cpp @@ -760,7 +760,7 @@ private: } - TVector<ui16> keyTypes; + TVector<NScheme::TTypeInfo> keyTypes; if (Self->Tables.contains(info->TablePathId)) { TTableInfo::TPtr tableInfo = Self->Tables.at(info->TablePathId); for (ui32 keyPos: tableInfo->KeyColumnIds) { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp index 75b65912fe1..275428e540c 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp @@ -2,6 +2,8 @@ #include "schemeshard__operation_common.h" #include "schemeshard_impl.h" +#include <ydb/core/scheme/scheme_types_proto.h> + namespace { using namespace NKikimr; @@ -57,7 +59,9 @@ TColumnTableInfo::TPtr ParseParams( for (const auto& col : tableSchema->GetColumns()) { ui32 id = col.GetId(); TString name = col.GetName(); - columns[id] = TOlapSchema::TColumn{id, name, static_cast<NScheme::TTypeId>(col.GetTypeId()), Max<ui32>()}; + auto typeInfo = NScheme::TypeInfoFromProtoColumnType(col.GetTypeId(), + col.HasTypeInfo() ? &col.GetTypeInfo() : nullptr); + columns[id] = TOlapSchema::TColumn{id, name, typeInfo, Max<ui32>()}; columnsByName[name] = id; } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp index d2a08fc11cf..bb9f55b837f 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp @@ -4,6 +4,7 @@ #include <ydb/core/engine/mkql_proto.h> #include <ydb/core/persqueue/writer/source_id_encoding.h> +#include <ydb/core/scheme/scheme_types_proto.h> #define LOG_D(stream) LOG_DEBUG_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) #define LOG_I(stream) LOG_INFO_S (context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) @@ -675,7 +676,11 @@ TVector<ISubOperationBase::TPtr> CreateNewCdcStream(TOperationId opId, const TTx auto& keyComponent = *pqConfig.AddPartitionKeySchema(); keyComponent.SetName(column.Name); - keyComponent.SetTypeId(column.PType); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.PType); + keyComponent.SetTypeId(columnType.TypeId); + if (columnType.TypeInfo) { + *keyComponent.MutableTypeInfo() = *columnType.TypeInfo; + } } auto& bootstrapConfig = *desc.MutableBootstrapConfig(); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_store.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_store.cpp index ffa94ef3191..a8b814ee5d4 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_store.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_store.cpp @@ -3,6 +3,7 @@ #include "schemeshard_impl.h" #include <ydb/core/base/subdomain.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/tx/columnshard/columnshard.h> #include <ydb/core/mind/hive/hive.h> @@ -56,12 +57,24 @@ bool PrepareSchema(NKikimrSchemeOp::TColumnTableSchema& proto, TOlapSchema& sche auto typeName = NMiniKQL::AdaptLegacyYqlType(colProto.GetType()); const NScheme::IType* type = typeRegistry->GetType(typeName); - if (!type || !NScheme::NTypeIds::IsYqlType(type->GetTypeId())) { - errStr = Sprintf("Type '%s' specified for column '%s' is not supported", colProto.GetType().c_str(), col.Name.c_str()); - return false; + if (type) { + if (!NScheme::NTypeIds::IsYqlType(type->GetTypeId())) { + errStr = Sprintf("Type '%s' specified for column '%s' is not supported", colProto.GetType().c_str(), col.Name.c_str()); + return false; + } + col.Type = NScheme::TTypeInfo(type->GetTypeId()); + } else { + auto* typeDesc = NPg::TypeDescFromPgTypeName(typeName); + if (!typeDesc) { + errStr = Sprintf("Type '%s' specified for column '%s' is not supported", colProto.GetType().c_str(), col.Name.c_str()); + } + col.Type = NScheme::TTypeInfo(NScheme::NTypeIds::Pg, typeDesc); + } + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(col.Type); + colProto.SetTypeId(columnType.TypeId); + if (columnType.TypeInfo) { + *colProto.MutableTypeInfo() = *columnType.TypeInfo; } - colProto.SetTypeId(type->GetTypeId()); - col.TypeId = type->GetTypeId(); if (schema.ColumnsByName.contains(col.Name)) { errStr = Sprintf("Duplicate column '%s'", col.Name.c_str()); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_table.cpp index 9ae12a29f7b..40e92d9f0fc 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_olap_table.cpp @@ -123,14 +123,21 @@ TColumnTableInfo::TPtr CreateColumnTable( auto typeName = NMiniKQL::AdaptLegacyYqlType(colProto.GetType()); const NScheme::IType* type = typeRegistry->GetType(typeName); + NScheme::TTypeInfo typeInfo; if (!type || !NScheme::NTypeIds::IsYqlType(type->GetTypeId())) { - status = NKikimrScheme::StatusSchemeError; - errStr = TStringBuilder() - << "Type '" << colProto.GetType() << "' specified for column '" << colName << "' is not supported"; - return nullptr; + auto* typeDesc = NPg::TypeDescFromPgTypeName(typeName); + if (!typeDesc) { + status = NKikimrScheme::StatusSchemeError; + errStr = TStringBuilder() + << "Type '" << colProto.GetType() << "' specified for column '" << colName << "' is not supported"; + return nullptr; + } + typeInfo = NScheme::TTypeInfo(NScheme::NTypeIds::Pg, typeDesc); + } else { + typeInfo = NScheme::TTypeInfo(type->GetTypeId()); } - if (type->GetTypeId() != col->TypeId) { + if (typeInfo != col->Type) { status = NKikimrScheme::StatusSchemeError; errStr = TStringBuilder() << "Type '" << colProto.GetType() << "' specified for column '" << colName diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp index 7dd80607321..df7e92d9691 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp @@ -44,7 +44,7 @@ bool CheckColumnTypesConstraints(NKikimrSchemeOp::TTableDescription& desc, TStri bool InitPartitioning(const NKikimrSchemeOp::TTableDescription& op, const NScheme::TTypeRegistry* typeRegistry, const TVector<ui32>& keyColIds, - const TVector<NScheme::TTypeId>& keyColTypeIds, + const TVector<NScheme::TTypeInfo>& keyColTypeIds, TString& errStr, TVector<TTableShardInfo>& partitions, const TSchemeLimits& limits) { @@ -63,7 +63,7 @@ bool InitPartitioning(const NKikimrSchemeOp::TTableDescription& op, TVector<TString> rangeEnds; if (op.HasUniformPartitionsCount()) { Y_VERIFY(!keyColIds.empty()); - NScheme::TTypeId firstKeyColType = keyColTypeIds[0]; + auto firstKeyColType = keyColTypeIds[0]; if (!TSchemeShard::FillUniformPartitioning(rangeEnds, keyColIds.size(), firstKeyColType, partitionCount, typeRegistry, errStr)) { return false; } @@ -106,17 +106,17 @@ bool DoInitPartitioning(TTableInfo::TPtr tableInfo, return false; } - TVector<NScheme::TTypeId> keyColTypeIds; + TVector<NScheme::TTypeInfo> keyColTypeIds; for (ui32 ki : keyColIds) { - NScheme::TTypeId typeId = tableInfo->Columns[ki].PType; + auto type = tableInfo->Columns[ki].PType; - if (!IsAllowedKeyType(typeId)) { + if (!IsAllowedKeyType(type)) { errStr = Sprintf("Column %s has wrong key type %s", - tableInfo->Columns[ki].Name.c_str(), NScheme::GetTypeName(typeId).c_str()); + tableInfo->Columns[ki].Name.c_str(), NScheme::TypeName(type)); return false; } - keyColTypeIds.push_back(typeId); + keyColTypeIds.push_back(type); } if (!InitPartitioning(op, typeRegistry, keyColIds, keyColTypeIds, errStr, partitions, limits)) { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp index ace2f95390d..ca1c88bc964 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_split_merge.cpp @@ -664,7 +664,7 @@ public: datashardInfo.BindedChannels = channels; // Build vector of key column types - TVector<NScheme::TTypeId> keyColTypeIds; + TVector<NScheme::TTypeInfo> keyColTypeIds; for (const auto& col : tableInfo->Columns) { if (!col.second.IsKey()) continue; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp index b83f4e60283..b3fa178cd91 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp @@ -4,6 +4,7 @@ #include "schemeshard_impl.h" #include <ydb/core/base/subdomain.h> +#include <ydb/core/scheme/scheme_types_proto.h> namespace { @@ -219,7 +220,11 @@ public: auto colDescr = descr.AddColumns(); colDescr->SetId(columnId); colDescr->SetName(column.Name); - colDescr->SetColType(column.PType); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.PType); + colDescr->SetColType(columnType.TypeId); + if (columnType.TypeInfo) { + *colDescr->MutableColTypeInfo() = *columnType.TypeInfo; + } colDescr->SetColKeyOrder(column.KeyOrder); colDescr->SetCreateVersion(column.CreateVersion); colDescr->SetDeleteVersion(column.DeleteVersion); diff --git a/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp b/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp index 1001f22c761..efdd914cec5 100644 --- a/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp @@ -5,8 +5,9 @@ namespace NKikimr { namespace NSchemeShard { -static bool IsIntegerType(NScheme::TTypeId typeId) { - switch (typeId) { +static bool IsIntegerType(NScheme::TTypeInfo type) { + // TODO: support pg types + switch (type.GetTypeId()) { case NScheme::NTypeIds::Bool: case NScheme::NTypeIds::Int8: @@ -29,7 +30,7 @@ static bool IsIntegerType(NScheme::TTypeId typeId) { } } -TSerializedCellVec ChooseSplitKeyByHistogram(const NKikimrTableStats::THistogram& histogram, const TConstArrayRef<NScheme::TTypeId> &keyColumnTypes) { +TSerializedCellVec ChooseSplitKeyByHistogram(const NKikimrTableStats::THistogram& histogram, const TConstArrayRef<NScheme::TTypeInfo> &keyColumnTypes) { ui64 bucketsCount = histogram.BucketsSize(); ui64 idxLo = bucketsCount * 0.33; ui64 idxMed = bucketsCount * 0.5; @@ -42,7 +43,7 @@ TSerializedCellVec ChooseSplitKeyByHistogram(const NKikimrTableStats::THistogram TVector<TCell> splitKey(keyMed.GetCells().size()); for (size_t i = 0; i < keyMed.GetCells().size(); ++i) { - NScheme::TTypeId columnType = keyColumnTypes[i]; + auto columnType = keyColumnTypes[i]; if (0 == CompareTypedCells(keyLo.GetCells()[i], keyHi.GetCells()[i], columnType)) { // lo == hi, so we add this value and proceed to the next column @@ -88,7 +89,7 @@ TSerializedCellVec ChooseSplitKeyByHistogram(const NKikimrTableStats::THistogram } TSerializedCellVec DoFindSplitKey(const TVector<std::pair<TSerializedCellVec, ui64>>& keysHist, - const TConstArrayRef<NScheme::TTypeId>& keyColumnTypes, + const TConstArrayRef<NScheme::TTypeInfo>& keyColumnTypes, const size_t prefixSize) { ui64 total = keysHist.back().second; @@ -123,7 +124,7 @@ TSerializedCellVec DoFindSplitKey(const TVector<std::pair<TSerializedCellVec, ui return TSerializedCellVec(TSerializedCellVec::Serialize(splitKey)); } -TSerializedCellVec ChooseSplitKeyByKeySample(const NKikimrTableStats::THistogram& keySample, const TConstArrayRef<NScheme::TTypeId>& keyColumnTypes) { +TSerializedCellVec ChooseSplitKeyByKeySample(const NKikimrTableStats::THistogram& keySample, const TConstArrayRef<NScheme::TTypeInfo>& keyColumnTypes) { TVector<std::pair<TSerializedCellVec, ui64>> keysHist; for (const auto& s : keySample.GetBuckets()) { keysHist.emplace_back(std::make_pair(TSerializedCellVec(s.GetKey()), s.GetValue())); @@ -327,7 +328,7 @@ bool TTxPartitionHistogram::Execute(TTransactionContext& txc, const TActorContex return true; } - TSmallVec<NScheme::TTypeId> keyColumnTypes(table->KeyColumnIds.size()); + TSmallVec<NScheme::TTypeInfo> keyColumnTypes(table->KeyColumnIds.size()); for (size_t ki = 0; ki < table->KeyColumnIds.size(); ++ki) { keyColumnTypes[ki] = table->Columns.FindPtr(table->KeyColumnIds[ki])->PType; } diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp index 50e484da109..1ca5ec73285 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp @@ -675,7 +675,7 @@ public: if (shardStatus.LastKeyAck) { //check that all LastKeyAcks are monotonously increase TTableInfo::TPtr tableInfo = Self->Tables.at(buildInfo->TablePathId); - TVector<ui16> keyTypes; + TVector<NScheme::TTypeInfo> keyTypes; for (ui32 keyPos: tableInfo->KeyColumnIds) { keyTypes.push_back(tableInfo->Columns.at(keyPos).PType); } diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index d22c965885f..bb35512dd97 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -9,6 +9,7 @@ #include <ydb/core/base/tx_processing.h> #include <ydb/core/engine/mkql_proto.h> #include <ydb/core/sys_view/partition_stats/partition_stats.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/library/yql/minikql/mkql_type_ops.h> namespace NKikimr { @@ -2288,10 +2289,16 @@ void TSchemeShard::PersistTableAltered(NIceDb::TNiceDb& db, const TPathId pathId for (auto col : tableInfo->Columns) { ui32 colId = col.first; const TTableInfo::TColumn& cinfo = col.second; + TString typeData; + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(cinfo.PType); + if (columnType.TypeInfo) { + Y_VERIFY(columnType.TypeInfo->SerializeToString(&typeData)); + } if (pathId.OwnerId == TabletID()) { db.Table<Schema::Columns>().Key(pathId.LocalPathId, colId).Update( NIceDb::TUpdate<Schema::Columns::ColName>(cinfo.Name), - NIceDb::TUpdate<Schema::Columns::ColType>((ui32)cinfo.PType), + NIceDb::TUpdate<Schema::Columns::ColType>((ui32)columnType.TypeId), + NIceDb::TUpdate<Schema::Columns::ColTypeData>(typeData), NIceDb::TUpdate<Schema::Columns::ColKeyOrder>(cinfo.KeyOrder), NIceDb::TUpdate<Schema::Columns::CreateVersion>(cinfo.CreateVersion), NIceDb::TUpdate<Schema::Columns::DeleteVersion>(cinfo.DeleteVersion), @@ -2304,7 +2311,8 @@ void TSchemeShard::PersistTableAltered(NIceDb::TNiceDb& db, const TPathId pathId } else { db.Table<Schema::MigratedColumns>().Key(pathId.OwnerId, pathId.LocalPathId, colId).Update( NIceDb::TUpdate<Schema::MigratedColumns::ColName>(cinfo.Name), - NIceDb::TUpdate<Schema::MigratedColumns::ColType>((ui32)cinfo.PType), + NIceDb::TUpdate<Schema::MigratedColumns::ColType>((ui32)columnType.TypeId), + NIceDb::TUpdate<Schema::MigratedColumns::ColTypeData>(typeData), NIceDb::TUpdate<Schema::MigratedColumns::ColKeyOrder>(cinfo.KeyOrder), NIceDb::TUpdate<Schema::MigratedColumns::CreateVersion>(cinfo.CreateVersion), NIceDb::TUpdate<Schema::MigratedColumns::DeleteVersion>(cinfo.DeleteVersion), @@ -2339,10 +2347,16 @@ void TSchemeShard::PersistAddAlterTable(NIceDb::TNiceDb& db, TPathId pathId, con for (auto col : alter->Columns) { ui32 colId = col.first; const TTableInfo::TColumn& cinfo = col.second; + TString typeData; + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(cinfo.PType); + if (columnType.TypeInfo) { + Y_VERIFY(columnType.TypeInfo->SerializeToString(&typeData)); + } if (pathId.OwnerId == TabletID()) { db.Table<Schema::ColumnAlters>().Key(pathId.LocalPathId, colId).Update( NIceDb::TUpdate<Schema::ColumnAlters::ColName>(cinfo.Name), - NIceDb::TUpdate<Schema::ColumnAlters::ColType>((ui32)cinfo.PType), + NIceDb::TUpdate<Schema::ColumnAlters::ColType>((ui32)columnType.TypeId), + NIceDb::TUpdate<Schema::ColumnAlters::ColTypeData>(typeData), NIceDb::TUpdate<Schema::ColumnAlters::ColKeyOrder>(cinfo.KeyOrder), NIceDb::TUpdate<Schema::ColumnAlters::CreateVersion>(cinfo.CreateVersion), NIceDb::TUpdate<Schema::ColumnAlters::DeleteVersion>(cinfo.DeleteVersion), @@ -2353,7 +2367,8 @@ void TSchemeShard::PersistAddAlterTable(NIceDb::TNiceDb& db, TPathId pathId, con } else { db.Table<Schema::MigratedColumnAlters>().Key(pathId.OwnerId, pathId.LocalPathId, colId).Update( NIceDb::TUpdate<Schema::MigratedColumnAlters::ColName>(cinfo.Name), - NIceDb::TUpdate<Schema::MigratedColumnAlters::ColType>((ui32)cinfo.PType), + NIceDb::TUpdate<Schema::MigratedColumnAlters::ColType>((ui32)columnType.TypeId), + NIceDb::TUpdate<Schema::MigratedColumnAlters::ColTypeData>(typeData), NIceDb::TUpdate<Schema::MigratedColumnAlters::ColKeyOrder>(cinfo.KeyOrder), NIceDb::TUpdate<Schema::MigratedColumnAlters::CreateVersion>(cinfo.CreateVersion), NIceDb::TUpdate<Schema::MigratedColumnAlters::DeleteVersion>(cinfo.DeleteVersion), @@ -5715,12 +5730,20 @@ TString TSchemeShard::FillAlterTableTxBody(TPathId pathId, TShardIdx shardIdx, T auto descr = proto->AddDropColumns(); descr->SetName(colInfo.Name); descr->SetId(colInfo.Id); - descr->SetTypeId(colInfo.PType); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(colInfo.PType); + descr->SetTypeId(columnType.TypeId); + if (columnType.TypeInfo) { + *descr->MutableTypeInfo() = *columnType.TypeInfo; + } } else { auto descr = proto->AddColumns(); descr->SetName(colInfo.Name); descr->SetId(colInfo.Id); - descr->SetTypeId(colInfo.PType); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(colInfo.PType); + descr->SetTypeId(columnType.TypeId); + if (columnType.TypeInfo) { + *descr->MutableTypeInfo() = *columnType.TypeInfo; + } descr->SetFamily(colInfo.Family); } } @@ -5742,7 +5765,7 @@ TString TSchemeShard::FillAlterTableTxBody(TPathId pathId, TShardIdx shardIdx, T return txBody; } -bool TSchemeShard::FillSplitPartitioning(TVector<TString>& rangeEnds, const TConstArrayRef<NScheme::TTypeId>& keyColTypes, +bool TSchemeShard::FillSplitPartitioning(TVector<TString>& rangeEnds, const TConstArrayRef<NScheme::TTypeInfo>& keyColTypes, const::google::protobuf::RepeatedPtrField<NKikimrSchemeOp::TSplitBoundary> &boundaries, TString& errStr) { for (int i = 0; i < boundaries.size(); ++i) { @@ -5899,7 +5922,7 @@ void TSchemeShard::FillTableDescription(TPathId tableId, ui32 partitionIdx, ui64 FillTableSchemaVersion(schemaVersion, tableDescr); } -bool TSchemeShard::FillUniformPartitioning(TVector<TString>& rangeEnds, ui32 keySize, NScheme::TTypeId firstKeyColType, ui32 partitionCount, const NScheme::TTypeRegistry* typeRegistry, TString& errStr) { +bool TSchemeShard::FillUniformPartitioning(TVector<TString>& rangeEnds, ui32 keySize, NScheme::TTypeInfo firstKeyColType, ui32 partitionCount, const NScheme::TTypeRegistry* typeRegistry, TString& errStr) { if (partitionCount > 1) { // RangeEnd key will have first cell with non-NULL value and rest of the cells with NULLs TVector<TCell> rangeEnd(keySize); @@ -5907,7 +5930,8 @@ bool TSchemeShard::FillUniformPartitioning(TVector<TString>& rangeEnds, ui32 key ui32 valSz = 0; // Check that first key column has integer type - switch(firstKeyColType) { + auto typeId = firstKeyColType.GetTypeId(); + switch(typeId) { case NScheme::NTypeIds::Uint32: maxVal = Max<ui32>(); valSz = 4; @@ -5917,7 +5941,7 @@ bool TSchemeShard::FillUniformPartitioning(TVector<TString>& rangeEnds, ui32 key valSz = 8; break; default: - errStr = TStringBuilder() << "Unsupported first key column type " << typeRegistry->GetTypeName(firstKeyColType) << ", only Uint32 and Uint64 are supported"; + errStr = TStringBuilder() << "Unsupported first key column type " << typeRegistry->GetTypeName(typeId) << ", only Uint32 and Uint64 are supported"; return false; } diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index 168998c3490..becaf324976 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -869,9 +869,9 @@ public: bool rangeBeginInclusive, bool rangeEndInclusive, bool newTable = false); void FillTableDescription(TPathId tableId, ui32 partitionIdx, ui64 schemaVersion, NKikimrSchemeOp::TTableDescription* tableDescr); - static bool FillUniformPartitioning(TVector<TString>& rangeEnds, ui32 keySize, NScheme::TTypeId firstKeyColType, + static bool FillUniformPartitioning(TVector<TString>& rangeEnds, ui32 keySize, NScheme::TTypeInfo firstKeyColType, ui32 partitionCount, const NScheme::TTypeRegistry* typeRegistry, TString& errStr); - static bool FillSplitPartitioning(TVector<TString>& rangeEnds, const TConstArrayRef<NScheme::TTypeId>& keyColTypes, + static bool FillSplitPartitioning(TVector<TString>& rangeEnds, const TConstArrayRef<NScheme::TTypeInfo>& keyColTypes, const ::google::protobuf::RepeatedPtrField<NKikimrSchemeOp::TSplitBoundary>& boundaries, TString& errStr); diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp index 5fcd1204e8a..01985b866ce 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp @@ -6,6 +6,7 @@ #include <ydb/core/base/tx_processing.h> #include <ydb/core/engine/minikql/flat_local_tx_factory.h> #include <ydb/core/engine/mkql_proto.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/tablet/tablet_counters_aggregator.h> #include <ydb/core/tablet/tablet_counters_protobuf.h> #include <ydb/core/util/pb.h> @@ -120,15 +121,21 @@ TTableInfo::TAlterDataPtr TTableInfo::CreateAlterData( return nullptr; } - if (!type) { - errStr = Sprintf("Type '%s' specified for column '%s' is not supported by storage", col.GetType().data(), colName.data()); - return nullptr; - } - - // Only allow YQL types - if (!NScheme::NTypeIds::IsYqlType(type->GetTypeId())) { - errStr = Sprintf("Type '%s' specified for column '%s' is no longer supported", col.GetType().data(), colName.data()); - return nullptr; + NScheme::TTypeInfo typeInfo; + if (type) { + // Only allow YQL types + if (!NScheme::NTypeIds::IsYqlType(type->GetTypeId())) { + errStr = Sprintf("Type '%s' specified for column '%s' is no longer supported", col.GetType().data(), colName.data()); + return nullptr; + } + typeInfo = NScheme::TTypeInfo(type->GetTypeId()); + } else { + auto* typeDesc = NPg::TypeDescFromPgTypeName(typeName); + if (!typeDesc) { + errStr = Sprintf("Type '%s' specified for column '%s' is not supported by storage", col.GetType().data(), colName.data()); + return nullptr; + } + typeInfo = NScheme::TTypeInfo(NScheme::NTypeIds::Pg, typeDesc); } ui32 colId = col.HasId() ? col.GetId() : alterData->NextColumnId; @@ -146,7 +153,7 @@ TTableInfo::TAlterDataPtr TTableInfo::CreateAlterData( colName2Id[colName] = colId; TTableInfo::TColumn& column = alterData->Columns[colId]; - column = TTableInfo::TColumn(colName, colId, type->GetTypeId()); + column = TTableInfo::TColumn(colName, colId, typeInfo); column.Family = columnFamily ? columnFamily->GetId() : 0; column.NotNull = col.GetNotNull(); if (source) @@ -310,7 +317,11 @@ TVector<ui32> TTableInfo::FillDescriptionCache(TPathElement::TPtr pathInfo) { auto colDescr = TableDescription.AddColumns(); colDescr->SetName(column.Name); colDescr->SetId(column.Id); - colDescr->SetTypeId(column.PType); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.PType); + colDescr->SetTypeId(columnType.TypeId); + if (columnType.TypeInfo) { + *colDescr->MutableTypeInfo() = *columnType.TypeInfo; + } colDescr->SetFamily(column.Family); } for (auto ci : keyColumnIds) { @@ -1836,14 +1847,15 @@ bool TPersQueueGroupInfo::FillKeySchema(const NKikimrPQ::TPQTabletConfig& tablet KeySchema.reserve(tabletConfig.PartitionKeySchemaSize()); for (const auto& component : tabletConfig.GetPartitionKeySchema()) { - if (!NScheme::NTypeIds::IsYqlType(component.GetTypeId())) { + // TODO: support pg types + auto typeId = component.GetTypeId(); + if (!NScheme::NTypeIds::IsYqlType(typeId)) { error = TStringBuilder() << "TypeId is not supported" - << ": typeId# " << component.GetTypeId() + << ": typeId# " << typeId << ", component# " << component.GetName(); return false; } - - KeySchema.push_back(component.GetTypeId()); + KeySchema.push_back(NScheme::TTypeInfo(typeId)); } return true; @@ -1980,7 +1992,8 @@ TOlapStoreInfo::TOlapStoreInfo( auto& col = preset.Columns[colProto.GetId()]; col.Id = colProto.GetId(); col.Name = colProto.GetName(); - col.TypeId = colProto.GetTypeId(); + col.Type = NScheme::TypeInfoFromProtoColumnType(colProto.GetTypeId(), + colProto.HasTypeInfo() ? &colProto.GetTypeInfo() : nullptr); preset.ColumnsByName[col.Name] = col.Id; } for (const auto& keyName : presetProto.GetSchema().GetKeyColumnNames()) { diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 4595022733a..e9845105bd3 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -313,7 +313,7 @@ struct TTableInfo : public TSimpleRefCount<TTableInfo> { TString DefaultValue; bool NotNull = false; - TColumn(const TString& name, ui32 id, NScheme::TTypeId type) + TColumn(const TString& name, ui32 id, NScheme::TTypeInfo type) : NTable::TScheme::TColumn(name, id, type) , CreateVersion(0) , DeleteVersion(Max<ui64>()) @@ -789,7 +789,7 @@ struct TOlapSchema { struct TColumn { ui32 Id = Max<ui32>(); TString Name; - NScheme::TTypeId TypeId = 0; + NScheme::TTypeInfo Type; ui32 KeyOrder = Max<ui32>(); // TODO: per-column ACL? @@ -901,7 +901,7 @@ struct TColumnTableInfo : TSimpleRefCount<TColumnTableInfo> { struct TPQShardInfo : TSimpleRefCount<TPQShardInfo> { using TPtr = TIntrusivePtr<TPQShardInfo>; - using TKeySchema = TVector<NScheme::TTypeId>; + using TKeySchema = TVector<NScheme::TTypeInfo>; struct TKeyRange { TMaybe<TString> FromBound; diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp index 53406550528..85934c644a5 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp @@ -3,6 +3,7 @@ #include <ydb/core/base/appdata.h> #include <ydb/core/engine/mkql_proto.h> #include <ydb/core/protos/flat_tx_scheme.pb.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <util/stream/format.h> @@ -907,8 +908,12 @@ void TSchemeShard::DescribeTable(const TTableInfo::TPtr tableInfo, const NScheme auto colDescr = entry->AddColumns(); colDescr->SetName(cinfo.Name); - colDescr->SetType(typeRegistry->GetTypeName(cinfo.PType)); - colDescr->SetTypeId(cinfo.PType); + colDescr->SetType(typeRegistry->GetTypeName(cinfo.PType.GetTypeId())); // TODO: no pg type details in string type + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(cinfo.PType); + colDescr->SetTypeId(columnType.TypeId); + if (columnType.TypeInfo) { + *colDescr->MutableTypeInfo() = *columnType.TypeInfo; + } colDescr->SetId(cinfo.Id); colDescr->SetNotNull(cinfo.NotNull); @@ -1121,8 +1126,8 @@ void TSchemeShard::FillTableBoundaries(const TTableInfo::TPtr tableInfo, google: auto boundary = boundaries.Add()->MutableKeyPrefix(); for (ui32 ki = 0; ki < endKey.GetCells().size(); ++ki){ const auto& c = endKey.GetCells()[ki]; - ui32 typeId = tableInfo->Columns[tableInfo->KeyColumnIds[ki]].PType; - bool ok = NMiniKQL::CellToValue(typeId, c, *boundary->AddTuple(), errStr); + auto type = tableInfo->Columns[tableInfo->KeyColumnIds[ki]].PType; + bool ok = NMiniKQL::CellToValue(type, c, *boundary->AddTuple(), errStr); Y_VERIFY(ok, "Failed to build key tuple at postition %" PRIu32 " error: %s", ki, errStr.data()); } } diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h index fce0f91dc31..76160368e86 100644 --- a/ydb/core/tx/schemeshard/schemeshard_schema.h +++ b/ydb/core/tx/schemeshard/schemeshard_schema.h @@ -167,6 +167,7 @@ struct Schema : NIceDb::Schema { struct ColId : Column<2, NScheme::NTypeIds::Uint32> {}; struct ColName : Column<3, NScheme::NTypeIds::Utf8> {}; struct ColType : Column<4, NScheme::NTypeIds::Uint32> {}; + struct ColTypeData : Column<12, NScheme::NTypeIds::String> {}; struct ColKeyOrder : Column<5, NScheme::NTypeIds::Uint32> {}; struct CreateVersion : Column<6, NScheme::NTypeIds::Uint64> {}; struct DeleteVersion : Column<7, NScheme::NTypeIds::Uint64> {}; @@ -178,7 +179,7 @@ struct Schema : NIceDb::Schema { using TKey = TableKey<TabId, ColId>; using TColumns = TableColumns<TabId, ColId, ColName, ColType, ColKeyOrder, CreateVersion, DeleteVersion, - Family, DefaultKind, DefaultValue, NotNull>; + Family, DefaultKind, DefaultValue, NotNull, ColTypeData>; }; struct MigratedColumns : Table<55> { @@ -188,6 +189,7 @@ struct Schema : NIceDb::Schema { struct ColName : Column<4, NScheme::NTypeIds::Utf8> {}; struct ColType : Column<5, NScheme::NTypeIds::Uint32> {}; + struct ColTypeData : Column<13, NScheme::NTypeIds::String> {}; struct ColKeyOrder : Column<6, NScheme::NTypeIds::Uint32> {}; struct CreateVersion : Column<7, NScheme::NTypeIds::Uint64> {}; struct DeleteVersion : Column<8, NScheme::NTypeIds::Uint64> {}; @@ -199,7 +201,7 @@ struct Schema : NIceDb::Schema { using TKey = TableKey<OwnerPathId, LocalPathId, ColId>; using TColumns = TableColumns<OwnerPathId, LocalPathId, ColId, ColName, ColType, ColKeyOrder, CreateVersion, DeleteVersion, - Family, DefaultKind, DefaultValue, NotNull>; + Family, DefaultKind, DefaultValue, NotNull, ColTypeData>; }; struct ColumnAlters : Table<13> { @@ -207,6 +209,7 @@ struct Schema : NIceDb::Schema { struct ColId : Column<2, NScheme::NTypeIds::Uint32> {}; struct ColName : Column<3, NScheme::NTypeIds::Utf8> {}; struct ColType : Column<4, NScheme::NTypeIds::Uint32> {}; + struct ColTypeData : Column<12, NScheme::NTypeIds::String> {}; struct ColKeyOrder : Column<5, NScheme::NTypeIds::Uint32> {}; struct CreateVersion : Column<6, NScheme::NTypeIds::Uint64> {}; struct DeleteVersion : Column<7, NScheme::NTypeIds::Uint64> {}; @@ -217,7 +220,7 @@ struct Schema : NIceDb::Schema { using TKey = TableKey<TabId, ColId>; using TColumns = TableColumns<TabId, ColId, ColName, ColType, ColKeyOrder, CreateVersion, DeleteVersion, - Family, DefaultKind, DefaultValue, NotNull>; + Family, DefaultKind, DefaultValue, NotNull, ColTypeData>; }; struct MigratedColumnAlters : Table<63> { @@ -227,6 +230,7 @@ struct Schema : NIceDb::Schema { struct ColName : Column<4, NScheme::NTypeIds::Utf8> {}; struct ColType : Column<5, NScheme::NTypeIds::Uint32> {}; + struct ColTypeData : Column<13, NScheme::NTypeIds::String> {}; struct ColKeyOrder : Column<6, NScheme::NTypeIds::Uint32> {}; struct CreateVersion : Column<7, NScheme::NTypeIds::Uint64> {}; struct DeleteVersion : Column<8, NScheme::NTypeIds::Uint64> {}; @@ -237,7 +241,7 @@ struct Schema : NIceDb::Schema { using TKey = TableKey<OwnerPathId, LocalPathId, ColId>; using TColumns = TableColumns<OwnerPathId, LocalPathId, ColId, ColName, ColType, ColKeyOrder, CreateVersion, DeleteVersion, - Family, DefaultKind, DefaultValue, NotNull>; + Family, DefaultKind, DefaultValue, NotNull, ColTypeData>; }; struct Shards : Table<7> { diff --git a/ydb/core/tx/schemeshard/schemeshard_utils.cpp b/ydb/core/tx/schemeshard/schemeshard_utils.cpp index e4ea14dfa40..f061c18e03d 100644 --- a/ydb/core/tx/schemeshard/schemeshard_utils.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_utils.cpp @@ -289,7 +289,9 @@ NKikimrSchemeOp::TTableDescription CalcImplTableDesc( auto item = result.AddColumns(); item->SetName(column.Name); - item->SetType(appData->TypeRegistry->GetTypeName(column.PType)); + // TODO: support pg types + Y_VERIFY(column.PType.GetTypeId() != NScheme::NTypeIds::Pg); + item->SetType(appData->TypeRegistry->GetTypeName(column.PType.GetTypeId())); ui32 order = Max<ui32>(); if (implKeyToImplColumn.contains(column.Name)) { @@ -473,7 +475,7 @@ NKikimrSchemeOp::TPartitionConfig PartitionConfigForIndexes( return PartitionConfigForIndexes(baseTableDesrc.GetPartitionConfig(), indexTableDesc); } -bool ExtractTypes(const NKikimrSchemeOp::TTableDescription& baseTableDesrc, TColumnTypes& columsTypes, TString& explain) { +bool ExtractTypes(const NKikimrSchemeOp::TTableDescription& baseTableDesrc, TColumnTypes& columnTypes, TString& explain) { const NScheme::TTypeRegistry* typeRegistry = AppData()->TypeRegistry; Y_VERIFY(typeRegistry); @@ -482,11 +484,15 @@ bool ExtractTypes(const NKikimrSchemeOp::TTableDescription& baseTableDesrc, TCol auto typeName = NMiniKQL::AdaptLegacyYqlType(column.GetType()); const NScheme::IType* type = typeRegistry->GetType(typeName); if (!type) { - explain += TStringBuilder() << "Type '" << column.GetType() << "' specified for column '" << columnName << "' is not supported by storage"; - return false; + auto* typeDesc = NPg::TypeDescFromPgTypeName(typeName); + if (!typeDesc) { + explain += TStringBuilder() << "Type '" << column.GetType() << "' specified for column '" << columnName << "' is not supported by storage"; + return false; + } + columnTypes[columnName] = NScheme::TTypeInfo(NScheme::NTypeIds::Pg, typeDesc); + } else { + columnTypes[columnName] = NScheme::TTypeInfo(type->GetTypeId()); } - auto typeId = type->GetTypeId(); - columsTypes[columnName] = typeId; } return true; @@ -503,7 +509,7 @@ TColumnTypes ExtractTypes(const NSchemeShard::TTableInfo::TPtr& baseTableInfo) { } bool IsCompatibleKeyTypes( - const TColumnTypes& baseTableColumsTypes, + const TColumnTypes& baseTableColumnTypes, const TTableColumns& implTableColumns, bool uniformTable, TString& explain) @@ -511,41 +517,49 @@ bool IsCompatibleKeyTypes( const NScheme::TTypeRegistry* typeRegistry = AppData()->TypeRegistry; Y_VERIFY(typeRegistry); - for (const auto& item: baseTableColumsTypes) { + for (const auto& item: baseTableColumnTypes) { auto& columnName = item.first; - auto& typeId = item.second; + auto typeId = item.second.GetTypeId(); - auto typeSP = typeRegistry->GetType(typeId); - if (!typeSP) { - explain += TStringBuilder() << "unknown typeId '" << typeId << "' for column '" << columnName << "'"; - return false; - } + if (typeId == NScheme::NTypeIds::Pg) { + if (!item.second.GetTypeDesc()) { + explain += TStringBuilder() << "unknown pg type for column '" << columnName << "'"; + return false; + } - if (!NScheme::NTypeIds::IsYqlType(typeId)) { - explain += TStringBuilder() << "Type '" << typeId << "' specified for column '" << columnName << "' is no longer supported"; - return false; + } else { + auto typeSP = typeRegistry->GetType(typeId); + if (!typeSP) { + explain += TStringBuilder() << "unknown typeId '" << typeId << "' for column '" << columnName << "'"; + return false; + } + + if (!NScheme::NTypeIds::IsYqlType(typeId)) { + explain += TStringBuilder() << "Type '" << typeId << "' specified for column '" << columnName << "' is no longer supported"; + return false; + } } } for (auto& keyName: implTableColumns.Keys) { - Y_VERIFY(baseTableColumsTypes.contains(keyName)); - auto typeId = baseTableColumsTypes.at(keyName); + Y_VERIFY(baseTableColumnTypes.contains(keyName)); + auto typeInfo = baseTableColumnTypes.at(keyName); if (uniformTable) { - switch (typeId) { + switch (typeInfo.GetTypeId()) { case NScheme::NTypeIds::Uint32: case NScheme::NTypeIds::Uint64: break; default: explain += TStringBuilder() << "Column '" << keyName << "' has wrong key type " - << NScheme::GetTypeName(typeId) << " for being key of table with uniform partitioning"; + << NScheme::TypeName(typeInfo) << " for being key of table with uniform partitioning"; return false; } } - if (!NSchemeShard::IsAllowedKeyType(typeId)) { - explain += TStringBuilder() << "Column '" << keyName << "' has wrong key type " << NScheme::GetTypeName(typeId) << " for being key"; + if (!NSchemeShard::IsAllowedKeyType(typeInfo)) { + explain += TStringBuilder() << "Column '" << keyName << "' has wrong key type " << NScheme::TypeName(typeInfo) << " for being key"; return false; } } diff --git a/ydb/core/tx/schemeshard/schemeshard_utils.h b/ydb/core/tx/schemeshard/schemeshard_utils.h index de7d17f316f..684b3f94ef9 100644 --- a/ydb/core/tx/schemeshard/schemeshard_utils.h +++ b/ydb/core/tx/schemeshard/schemeshard_utils.h @@ -21,14 +21,16 @@ namespace NKikimr { namespace NSchemeShard { -inline bool IsAllowedKeyType(NScheme::TTypeId typeId) { - switch (typeId) { +inline bool IsAllowedKeyType(NScheme::TTypeInfo typeInfo) { + switch (typeInfo.GetTypeId()) { case NScheme::NTypeIds::Json: case NScheme::NTypeIds::Yson: case NScheme::NTypeIds::Float: case NScheme::NTypeIds::Double: case NScheme::NTypeIds::JsonDocument: return false; + case NScheme::NTypeIds::Pg: + return NPg::TypeDescIsComparable(typeInfo.GetTypeDesc()); default: return true; } @@ -52,7 +54,7 @@ inline NKikimrSchemeOp::TModifyScheme TransactionTemplate(const TString& working } TSerializedCellVec ChooseSplitKeyByHistogram(const NKikimrTableStats::THistogram& histogram, - const TConstArrayRef<NScheme::TTypeId>& keyColumnTypes); + const TConstArrayRef<NScheme::TTypeInfo>& keyColumnTypes); class TShardDeleter { struct TPerHiveDeletions { @@ -147,7 +149,7 @@ TTableColumns ExtractInfo(const NKikimrSchemeOp::TTableDescription& tableDesrc); TIndexColumns ExtractInfo(const NKikimrSchemeOp::TIndexCreationConfig& indexDesc); TTableColumns ExtractInfo(const NSchemeShard::TTableInfo::TPtr& tableInfo); -using TColumnTypes = THashMap<TString, NScheme::TTypeId>; +using TColumnTypes = THashMap<TString, NScheme::TTypeInfo>; bool ExtractTypes(const NKikimrSchemeOp::TTableDescription& baseTableDesrc, TColumnTypes& columsTypes, TString& explain); TColumnTypes ExtractTypes(const NSchemeShard::TTableInfo::TPtr& baseTableInfo); diff --git a/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp b/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp index 58cc395905c..b209f7e43c1 100644 --- a/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp @@ -14,7 +14,8 @@ namespace { inline ui32 GetType(const TOlapSchema::TColumn& col) { - return col.TypeId; + Y_VERIFY(col.Type.GetTypeId() != NScheme::NTypeIds::Pg, "pg types are not supported"); + return col.Type.GetTypeId(); } inline @@ -24,7 +25,8 @@ namespace { inline ui32 GetType(const TTableInfo::TColumn& col) { - return col.PType; + Y_VERIFY(col.PType.GetTypeId() != NScheme::NTypeIds::Pg, "pg types are not supported"); + return col.PType.GetTypeId(); } } @@ -144,7 +146,7 @@ static bool ValidateColumnTableTtl(const NKikimrSchemeOp::TColumnDataLifeCycle:: } // TODO: Support TTL for types other than Timestamp - if (alterColumns.FindPtr(colId)->TypeId != NScheme::NTypeIds::Timestamp) { + if (alterColumns.FindPtr(colId)->Type.GetTypeId() != NScheme::NTypeIds::Timestamp) { errStr = "Currently TTL is only supported on columns of Timestamp type"; return false; } diff --git a/ydb/core/tx/schemeshard/ut_async_index/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_async_index/CMakeLists.darwin.txt index 8a721b87617..68213fe33e4 100644 --- a/ydb/core/tx/schemeshard/ut_async_index/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_async_index/CMakeLists.darwin.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_async_index PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main core-tx-schemeshard - ydb-core-testlib + core-testlib-default tx-schemeshard-ut_helpers ) target_link_options(ydb-core-tx-schemeshard-ut_async_index PRIVATE diff --git a/ydb/core/tx/schemeshard/ut_async_index/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_async_index/CMakeLists.linux.txt index 3308fce5eab..8dd7481038c 100644 --- a/ydb/core/tx/schemeshard/ut_async_index/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_async_index/CMakeLists.linux.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_async_index PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main core-tx-schemeshard - ydb-core-testlib + core-testlib-default tx-schemeshard-ut_helpers ) target_link_options(ydb-core-tx-schemeshard-ut_async_index PRIVATE diff --git a/ydb/core/tx/schemeshard/ut_backup/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_backup/CMakeLists.darwin.txt index 74529d7845c..ac9af780e94 100644 --- a/ydb/core/tx/schemeshard/ut_backup/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_backup/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_backup PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers core-wrappers-ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_backup/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_backup/CMakeLists.linux.txt index 0215b366f17..ab326902116 100644 --- a/ydb/core/tx/schemeshard/ut_backup/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_backup/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_backup PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers core-wrappers-ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_base.cpp b/ydb/core/tx/schemeshard/ut_base.cpp index 970fdf18808..8c147d0f113 100644 --- a/ydb/core/tx/schemeshard/ut_base.cpp +++ b/ydb/core/tx/schemeshard/ut_base.cpp @@ -9669,13 +9669,13 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { } - class TSchemaHelpler { + class TSchemaHelper { private: NScheme::TTypeRegistry TypeRegistry; - const TVector<NKikimr::NScheme::TTypeId> KeyColumnTypes; + const TVector<NKikimr::NScheme::TTypeInfo> KeyColumnTypes; public: - explicit TSchemaHelpler(const TArrayRef<NKikimr::NScheme::TTypeId>& keyColumnTypes) + explicit TSchemaHelper(const TArrayRef<NKikimr::NScheme::TTypeInfo>& keyColumnTypes) : KeyColumnTypes(keyColumnTypes.begin(), keyColumnTypes.end()) {} @@ -9694,7 +9694,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { if (tuple[i] == "NULL") { cells.push_back(NKikimr::TCell()); } else { - switch (KeyColumnTypes[i]) { + switch (KeyColumnTypes[i].GetTypeId()) { #define ADD_CELL_FROM_STRING(ydbType, cppType) \ case NKikimr::NScheme::NTypeIds::ydbType: { \ cppType val = FromString<cppType>(tuple[i]); \ @@ -9750,13 +9750,13 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { }; Y_UNIT_TEST(SplitKey) { - TSmallVec<NScheme::TTypeId> keyColumnTypes = { - NScheme::NTypeIds::Uint64, - NScheme::NTypeIds::Utf8, - NScheme::NTypeIds::Uint32 + TSmallVec<NScheme::TTypeInfo> keyColumnTypes = { + NScheme::TTypeInfo(NScheme::NTypeIds::Uint64), + NScheme::TTypeInfo(NScheme::NTypeIds::Utf8), + NScheme::TTypeInfo(NScheme::NTypeIds::Uint32) }; - TSchemaHelpler schemaHelper(keyColumnTypes); + TSchemaHelper schemaHelper(keyColumnTypes); { TString splitKey = schemaHelper.FindSplitKey({ diff --git a/ydb/core/tx/schemeshard/ut_base/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_base/CMakeLists.darwin.txt index f04138f6624..bfd7e936008 100644 --- a/ydb/core/tx/schemeshard/ut_base/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_base/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_base PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_base/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_base/CMakeLists.linux.txt index d50666ddb55..35beb7025ea 100644 --- a/ydb/core/tx/schemeshard/ut_base/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_base/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_base PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_base_reboots/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_base_reboots/CMakeLists.darwin.txt index ac31e91f777..0158c970a98 100644 --- a/ydb/core/tx/schemeshard/ut_base_reboots/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_base_reboots/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_base_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_base_reboots/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_base_reboots/CMakeLists.linux.txt index 06d449f6785..a6041ec8d62 100644 --- a/ydb/core/tx/schemeshard/ut_base_reboots/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_base_reboots/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_base_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_bsvolume/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_bsvolume/CMakeLists.darwin.txt index 9ccf408a862..894fa9e2d31 100644 --- a/ydb/core/tx/schemeshard/ut_bsvolume/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_bsvolume/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_bsvolume PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_bsvolume/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_bsvolume/CMakeLists.linux.txt index f9d5d12d7f3..00b04b97bb9 100644 --- a/ydb/core/tx/schemeshard/ut_bsvolume/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_bsvolume/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_bsvolume PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_bsvolume_reboots/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_bsvolume_reboots/CMakeLists.darwin.txt index ee5ca11e275..67e5667c4dd 100644 --- a/ydb/core/tx/schemeshard/ut_bsvolume_reboots/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_bsvolume_reboots/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_bsvolume_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_bsvolume_reboots/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_bsvolume_reboots/CMakeLists.linux.txt index 35cb6d7e695..bf6a0f91439 100644 --- a/ydb/core/tx/schemeshard/ut_bsvolume_reboots/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_bsvolume_reboots/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_bsvolume_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_cdc_stream/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_cdc_stream/CMakeLists.darwin.txt index 1af32c35713..9d5d5728dda 100644 --- a/ydb/core/tx/schemeshard/ut_cdc_stream/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_cdc_stream/CMakeLists.darwin.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_cdc_stream PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main core-tx-schemeshard - ydb-core-testlib + core-testlib-default tx-schemeshard-ut_helpers library-cpp-json ) diff --git a/ydb/core/tx/schemeshard/ut_cdc_stream/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_cdc_stream/CMakeLists.linux.txt index 7a6f682acd6..219485e8761 100644 --- a/ydb/core/tx/schemeshard/ut_cdc_stream/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_cdc_stream/CMakeLists.linux.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_cdc_stream PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main core-tx-schemeshard - ydb-core-testlib + core-testlib-default tx-schemeshard-ut_helpers library-cpp-json ) diff --git a/ydb/core/tx/schemeshard/ut_cdc_stream_reboots/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_cdc_stream_reboots/CMakeLists.darwin.txt index 94e0ccd1a57..6fb92e07991 100644 --- a/ydb/core/tx/schemeshard/ut_cdc_stream_reboots/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_cdc_stream_reboots/CMakeLists.darwin.txt @@ -21,6 +21,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_cdc_stream_reboots PUBLIC cpp-testing-unittest_main core-tx-schemeshard tx-schemeshard-ut_helpers + yql-sql-pg_dummy ) target_link_options(ydb-core-tx-schemeshard-ut_cdc_stream_reboots PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/tx/schemeshard/ut_cdc_stream_reboots/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_cdc_stream_reboots/CMakeLists.linux.txt index 7c4e1c70df0..395b3314b32 100644 --- a/ydb/core/tx/schemeshard/ut_cdc_stream_reboots/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_cdc_stream_reboots/CMakeLists.linux.txt @@ -23,6 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_cdc_stream_reboots PUBLIC cpp-testing-unittest_main core-tx-schemeshard tx-schemeshard-ut_helpers + yql-sql-pg_dummy ) target_link_options(ydb-core-tx-schemeshard-ut_cdc_stream_reboots PRIVATE -ldl diff --git a/ydb/core/tx/schemeshard/ut_compaction/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_compaction/CMakeLists.darwin.txt index c543359def6..f1e290382ad 100644 --- a/ydb/core/tx/schemeshard/ut_compaction/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_compaction/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_compaction PUBLIC library-cpp-getopt cpp-regex-pcre ydb-core-cms - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers core-wrappers-ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_compaction/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_compaction/CMakeLists.linux.txt index 48cccd9ad29..608ddf90b77 100644 --- a/ydb/core/tx/schemeshard/ut_compaction/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_compaction/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_compaction PUBLIC library-cpp-getopt cpp-regex-pcre ydb-core-cms - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers core-wrappers-ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_export/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_export/CMakeLists.darwin.txt index 3a423d8fe6f..ae1c2216fac 100644 --- a/ydb/core/tx/schemeshard/ut_export/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_export/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_export PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers core-wrappers-ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_export/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_export/CMakeLists.linux.txt index c008c18d915..3611928a1b1 100644 --- a/ydb/core/tx/schemeshard/ut_export/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_export/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_export PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers core-wrappers-ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_export_reboots_s3/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_export_reboots_s3/CMakeLists.darwin.txt index 41206c76ee8..364a12f0697 100644 --- a/ydb/core/tx/schemeshard/ut_export_reboots_s3/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_export_reboots_s3/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_export_reboots_s3 PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers core-wrappers-ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_export_reboots_s3/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_export_reboots_s3/CMakeLists.linux.txt index 070065da3ca..0ae207499f3 100644 --- a/ydb/core/tx/schemeshard/ut_export_reboots_s3/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_export_reboots_s3/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_export_reboots_s3 PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers core-wrappers-ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_extsubdomain/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_extsubdomain/CMakeLists.darwin.txt index 4811a8c1d8b..887d3425ed2 100644 --- a/ydb/core/tx/schemeshard/ut_extsubdomain/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_extsubdomain/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_extsubdomain PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_extsubdomain/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_extsubdomain/CMakeLists.linux.txt index 69f6d6915b1..8b7c447658f 100644 --- a/ydb/core/tx/schemeshard/ut_extsubdomain/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_extsubdomain/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_extsubdomain PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_extsubdomain_reboots/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_extsubdomain_reboots/CMakeLists.darwin.txt index a9c3594c7c9..18b25bdbb28 100644 --- a/ydb/core/tx/schemeshard/ut_extsubdomain_reboots/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_extsubdomain_reboots/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_extsubdomain_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_extsubdomain_reboots/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_extsubdomain_reboots/CMakeLists.linux.txt index d1709e44b38..1a30ea27997 100644 --- a/ydb/core/tx/schemeshard/ut_extsubdomain_reboots/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_extsubdomain_reboots/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_extsubdomain_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_filestore_reboots/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_filestore_reboots/CMakeLists.darwin.txt index d63aa53ae92..a43e52b9eba 100644 --- a/ydb/core/tx/schemeshard/ut_filestore_reboots/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_filestore_reboots/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_filestore_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_filestore_reboots/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_filestore_reboots/CMakeLists.linux.txt index 5a84a7adf55..60502168982 100644 --- a/ydb/core/tx/schemeshard/ut_filestore_reboots/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_filestore_reboots/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_filestore_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp index 66d2b93f437..f0558cacd95 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp @@ -2019,7 +2019,7 @@ namespace NSchemeShardUT_Private { keyIdx = ki; } } - table.Columns.insert(std::make_pair(c.GetName(), TColumn{c.GetId(), keyIdx, c.GetTypeId(), 0, + table.Columns.insert(std::make_pair(c.GetName(), TColumn{c.GetId(), keyIdx, NScheme::TTypeInfo(c.GetTypeId()), 0, EColumnTypeConstraint::Nullable})); } dbSchemeResolver.AddTable(table); @@ -2036,17 +2036,16 @@ namespace NSchemeShardUT_Private { const auto& tdesc = pathDesc.GetTable(); // Cout << pathDesc << Endl; - std::unordered_map<ui32, NScheme::TTypeId> columnId2Type; + std::unordered_map<ui32, NScheme::TTypeInfo> columnId2Type; for (size_t i = 0; i < tdesc.ColumnsSize(); ++i) { ui32 id = tdesc.GetColumns(i).GetId(); - ui16 typeId = tdesc.GetColumns(i).GetTypeId(); - columnId2Type[id] = typeId; + auto typeInfo = NScheme::TTypeInfo(tdesc.GetColumns(i).GetTypeId()); + columnId2Type[id] = typeInfo; } for (size_t i = 0; i < tdesc.KeyColumnIdsSize(); ++i) { ui32 id = tdesc.GetKeyColumnIds(i); - NScheme::TTypeId typeId = columnId2Type[id]; - TablePartitioningInfo.KeyColumnTypes.push_back(typeId); + TablePartitioningInfo.KeyColumnTypes.push_back(columnId2Type[id]); } for (size_t i = 0; i < pathDesc.TablePartitionsSize(); ++i) { diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.h b/ydb/core/tx/schemeshard/ut_helpers/helpers.h index 78701f59a77..75243ca77f1 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/helpers.h +++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.h @@ -450,7 +450,7 @@ namespace NSchemeShardUT_Private { ui64 Datashard = 0; }; - TVector<NScheme::TTypeId> KeyColumnTypes; + TVector<NScheme::TTypeInfo> KeyColumnTypes; TVector<TBorder> Partitioning; std::shared_ptr<const TVector<TKeyDesc::TPartitionInfo>> ResolveKey(const TTableRange& range) const; diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp index 8a5bb30c13f..9fe0d725e4a 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp @@ -3,6 +3,7 @@ #include <ydb/core/engine/mkql_proto.h> #include <ydb/core/scheme/scheme_tablecell.h> #include <ydb/core/scheme/scheme_tabledefs.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/public/lib/scheme_types/scheme_type_id.h> #include <library/cpp/testing/unittest/registar.h> @@ -397,11 +398,12 @@ TCheckFunc CheckColumns(const TString& name, const TSet<TString>& columns, const void CheckBoundaries(const NKikimrScheme::TEvDescribeSchemeResult &record) { const NKikimrSchemeOp::TPathDescription& descr = record.GetPathDescription(); - THashMap<ui32, NScheme::TTypeId> colTypes; + THashMap<ui32, NScheme::TTypeInfo> colTypes; for (const auto& col : descr.GetTable().GetColumns()) { - colTypes[col.GetId()] = col.GetTypeId(); + colTypes[col.GetId()] = NScheme::TypeInfoFromProtoColumnType(col.GetTypeId(), + col.HasTypeInfo() ? &col.GetTypeInfo() : nullptr); } - TVector<NScheme::TTypeId> keyColTypes; + TVector<NScheme::TTypeInfo> keyColTypes; for (const auto& ki : descr.GetTable().GetKeyColumnIds()) { keyColTypes.push_back(colTypes[ki]); } diff --git a/ydb/core/tx/schemeshard/ut_index_build/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_index_build/CMakeLists.darwin.txt index a4cf64db66b..acabcfd0b46 100644 --- a/ydb/core/tx/schemeshard/ut_index_build/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_index_build/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_index_build PUBLIC cpp-regex-pcre library-cpp-svnversion ydb-core-metering - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers ) diff --git a/ydb/core/tx/schemeshard/ut_index_build/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_index_build/CMakeLists.linux.txt index 5ae1ec8563e..117b51a2352 100644 --- a/ydb/core/tx/schemeshard/ut_index_build/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_index_build/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_index_build PUBLIC cpp-regex-pcre library-cpp-svnversion ydb-core-metering - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers ) diff --git a/ydb/core/tx/schemeshard/ut_index_build_reboots/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_index_build_reboots/CMakeLists.darwin.txt index 749a4d73102..da78d417617 100644 --- a/ydb/core/tx/schemeshard/ut_index_build_reboots/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_index_build_reboots/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_index_build_reboots PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_index_build_reboots/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_index_build_reboots/CMakeLists.linux.txt index 0dcdb8d4ea2..411a9908289 100644 --- a/ydb/core/tx/schemeshard/ut_index_build_reboots/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_index_build_reboots/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_index_build_reboots PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_login/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_login/CMakeLists.darwin.txt index 8c525778628..64f3cd949fa 100644 --- a/ydb/core/tx/schemeshard/ut_login/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_login/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_login PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers ydb-library-login diff --git a/ydb/core/tx/schemeshard/ut_login/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_login/CMakeLists.linux.txt index 319242330a1..df29edc735b 100644 --- a/ydb/core/tx/schemeshard/ut_login/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_login/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_login PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers ydb-library-login diff --git a/ydb/core/tx/schemeshard/ut_move/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_move/CMakeLists.darwin.txt index b6838fd7b43..abca206eb11 100644 --- a/ydb/core/tx/schemeshard/ut_move/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_move/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_move PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_move/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_move/CMakeLists.linux.txt index d1253db6305..4d6ac392706 100644 --- a/ydb/core/tx/schemeshard/ut_move/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_move/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_move PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_move_reboots/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_move_reboots/CMakeLists.darwin.txt index 20552d41911..8f88cb38613 100644 --- a/ydb/core/tx/schemeshard/ut_move_reboots/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_move_reboots/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_move_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_move_reboots/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_move_reboots/CMakeLists.linux.txt index 1c83f488aa9..d49e9db3353 100644 --- a/ydb/core/tx/schemeshard/ut_move_reboots/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_move_reboots/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_move_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap.cpp index 0c4466dcac6..154f062b9ec 100644 --- a/ydb/core/tx/schemeshard/ut_olap.cpp +++ b/ydb/core/tx/schemeshard/ut_olap.cpp @@ -12,7 +12,7 @@ namespace NKikimr { namespace { namespace NTypeIds = NScheme::NTypeIds; -using TTypeId = NScheme::TTypeId; +using TTypeInfo = NScheme::TTypeInfo; static const TString defaultStoreSchema = R"( Name: "OlapStore" @@ -28,9 +28,9 @@ static const TString defaultStoreSchema = R"( } )"; -static const TVector<std::pair<TString, TTypeId>> defaultYdbSchema = { - {"timestamp", NTypeIds::Timestamp }, - {"data", NTypeIds::Utf8 } +static const TVector<std::pair<TString, TTypeInfo>> defaultYdbSchema = { + {"timestamp", TTypeInfo(NTypeIds::Timestamp) }, + {"data", TTypeInfo(NTypeIds::Utf8) } }; TString MakeTestBlob(std::pair<ui64, ui64> range) { @@ -39,7 +39,10 @@ TString MakeTestBlob(std::pair<ui64, ui64> range) { batchBuilder.Start(defaultYdbSchema, 0, 0, err); TString str; - TVector<TTypeId> types = {NTypeIds::Timestamp, NTypeIds::Utf8}; + TVector<TTypeInfo> types = { + TTypeInfo(NTypeIds::Timestamp), + TTypeInfo(NTypeIds::Utf8) + }; for (size_t i = range.first; i < range.second; ++i) { str = ToString(i); diff --git a/ydb/core/tx/schemeshard/ut_olap/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_olap/CMakeLists.darwin.txt index ec747dd579d..300270735bc 100644 --- a/ydb/core/tx/schemeshard/ut_olap/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_olap/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_olap PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-formats ydb-core-tx core-tx-columnshard diff --git a/ydb/core/tx/schemeshard/ut_olap/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_olap/CMakeLists.linux.txt index 470736e5f68..b0071a2c632 100644 --- a/ydb/core/tx/schemeshard/ut_olap/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_olap/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_olap PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-formats ydb-core-tx core-tx-columnshard diff --git a/ydb/core/tx/schemeshard/ut_olap_reboots/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_olap_reboots/CMakeLists.darwin.txt index 978796785bf..0d5e0f1768f 100644 --- a/ydb/core/tx/schemeshard/ut_olap_reboots/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_olap_reboots/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_olap_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx core-tx-columnshard tx-schemeshard-ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_olap_reboots/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_olap_reboots/CMakeLists.linux.txt index 7fd33fbdbc1..55d0fb0d54f 100644 --- a/ydb/core/tx/schemeshard/ut_olap_reboots/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_olap_reboots/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_olap_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx core-tx-columnshard tx-schemeshard-ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_pq/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_pq/CMakeLists.darwin.txt index 5fe075ac1bb..017d6cdf356 100644 --- a/ydb/core/tx/schemeshard/ut_pq/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_pq/CMakeLists.darwin.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_pq PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_pq/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_pq/CMakeLists.linux.txt index b0ffde8b157..2a84ea3ced5 100644 --- a/ydb/core/tx/schemeshard/ut_pq/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_pq/CMakeLists.linux.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_pq PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_pq_reboots/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_pq_reboots/CMakeLists.darwin.txt index db114cfb7ad..2d1d7f31e65 100644 --- a/ydb/core/tx/schemeshard/ut_pq_reboots/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_pq_reboots/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_pq_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_pq_reboots/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_pq_reboots/CMakeLists.linux.txt index 03388e8d696..1d74c762c75 100644 --- a/ydb/core/tx/schemeshard/ut_pq_reboots/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_pq_reboots/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_pq_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_reboots/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_reboots/CMakeLists.darwin.txt index baf3e802bf7..044408beda6 100644 --- a/ydb/core/tx/schemeshard/ut_reboots/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_reboots/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_reboots/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_reboots/CMakeLists.linux.txt index c33e1f3e248..a8d69a6c872 100644 --- a/ydb/core/tx/schemeshard/ut_reboots/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_reboots/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_replication/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_replication/CMakeLists.darwin.txt index b67dedfee33..e974358b977 100644 --- a/ydb/core/tx/schemeshard/ut_replication/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_replication/CMakeLists.darwin.txt @@ -21,6 +21,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_replication PUBLIC cpp-testing-unittest_main core-tx-schemeshard tx-schemeshard-ut_helpers + yql-sql-pg_dummy ) target_link_options(ydb-core-tx-schemeshard-ut_replication PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/tx/schemeshard/ut_replication/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_replication/CMakeLists.linux.txt index 327db0216ac..c60c53719f5 100644 --- a/ydb/core/tx/schemeshard/ut_replication/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_replication/CMakeLists.linux.txt @@ -23,6 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_replication PUBLIC cpp-testing-unittest_main core-tx-schemeshard tx-schemeshard-ut_helpers + yql-sql-pg_dummy ) target_link_options(ydb-core-tx-schemeshard-ut_replication PRIVATE -ldl diff --git a/ydb/core/tx/schemeshard/ut_replication_reboots/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_replication_reboots/CMakeLists.darwin.txt index d2ad8ae5c7b..41e5928e1cc 100644 --- a/ydb/core/tx/schemeshard/ut_replication_reboots/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_replication_reboots/CMakeLists.darwin.txt @@ -21,6 +21,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_replication_reboots PUBLIC cpp-testing-unittest_main core-tx-schemeshard tx-schemeshard-ut_helpers + yql-sql-pg_dummy ) target_link_options(ydb-core-tx-schemeshard-ut_replication_reboots PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/tx/schemeshard/ut_replication_reboots/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_replication_reboots/CMakeLists.linux.txt index 0b8510aeed7..e41d7cb085c 100644 --- a/ydb/core/tx/schemeshard/ut_replication_reboots/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_replication_reboots/CMakeLists.linux.txt @@ -23,6 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_replication_reboots PUBLIC cpp-testing-unittest_main core-tx-schemeshard tx-schemeshard-ut_helpers + yql-sql-pg_dummy ) target_link_options(ydb-core-tx-schemeshard-ut_replication_reboots PRIVATE -ldl diff --git a/ydb/core/tx/schemeshard/ut_restore/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_restore/CMakeLists.darwin.txt index d15fa7d2e66..32d1f0e0592 100644 --- a/ydb/core/tx/schemeshard/ut_restore/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_restore/CMakeLists.darwin.txt @@ -26,6 +26,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_restore PUBLIC tx-schemeshard-ut_helpers core-wrappers-ut_helpers ydb-core-ydb_convert + yql-sql-pg_dummy ) target_link_options(ydb-core-tx-schemeshard-ut_restore PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/tx/schemeshard/ut_restore/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_restore/CMakeLists.linux.txt index 1a1ee2dde23..66cac111b61 100644 --- a/ydb/core/tx/schemeshard/ut_restore/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_restore/CMakeLists.linux.txt @@ -28,6 +28,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_restore PUBLIC tx-schemeshard-ut_helpers core-wrappers-ut_helpers ydb-core-ydb_convert + yql-sql-pg_dummy ) target_link_options(ydb-core-tx-schemeshard-ut_restore PRIVATE -ldl diff --git a/ydb/core/tx/schemeshard/ut_rtmr/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_rtmr/CMakeLists.darwin.txt index b53e9b9f385..c882009debc 100644 --- a/ydb/core/tx/schemeshard/ut_rtmr/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_rtmr/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_rtmr PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_rtmr/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_rtmr/CMakeLists.linux.txt index 3d5cdcc8fe9..e10313e8fc0 100644 --- a/ydb/core/tx/schemeshard/ut_rtmr/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_rtmr/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_rtmr PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_rtmr_reboots/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_rtmr_reboots/CMakeLists.darwin.txt index b38fcbad8a6..b73503e073c 100644 --- a/ydb/core/tx/schemeshard/ut_rtmr_reboots/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_rtmr_reboots/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_rtmr_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_rtmr_reboots/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_rtmr_reboots/CMakeLists.linux.txt index 982cc4b34e9..59493549e25 100644 --- a/ydb/core/tx/schemeshard/ut_rtmr_reboots/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_rtmr_reboots/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_rtmr_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_sequence/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_sequence/CMakeLists.darwin.txt index fa9c56900e8..c8ca96dbc67 100644 --- a/ydb/core/tx/schemeshard/ut_sequence/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_sequence/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_sequence PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx core-tx-columnshard tx-schemeshard-ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_sequence/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_sequence/CMakeLists.linux.txt index 8837294a866..579c2fd2503 100644 --- a/ydb/core/tx/schemeshard/ut_sequence/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_sequence/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_sequence PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx core-tx-columnshard tx-schemeshard-ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_sequence_reboots/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_sequence_reboots/CMakeLists.darwin.txt index c43d0b09cc8..407211f9016 100644 --- a/ydb/core/tx/schemeshard/ut_sequence_reboots/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_sequence_reboots/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_sequence_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx core-tx-columnshard tx-schemeshard-ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_sequence_reboots/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_sequence_reboots/CMakeLists.linux.txt index 4eff1a7241b..ead5de8b158 100644 --- a/ydb/core/tx/schemeshard/ut_sequence_reboots/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_sequence_reboots/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_sequence_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx core-tx-columnshard tx-schemeshard-ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_serverless/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_serverless/CMakeLists.darwin.txt index d86ecb7de22..6168fe217f7 100644 --- a/ydb/core/tx/schemeshard/ut_serverless/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_serverless/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_serverless PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers ydb-core-yql_testlib diff --git a/ydb/core/tx/schemeshard/ut_serverless/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_serverless/CMakeLists.linux.txt index d664cdc3e8e..e9836aced1a 100644 --- a/ydb/core/tx/schemeshard/ut_serverless/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_serverless/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_serverless PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers ydb-core-yql_testlib diff --git a/ydb/core/tx/schemeshard/ut_split_merge/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_split_merge/CMakeLists.darwin.txt index 9dbeebfe903..cd264ac4882 100644 --- a/ydb/core/tx/schemeshard/ut_split_merge/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_split_merge/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_split_merge PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_split_merge/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_split_merge/CMakeLists.linux.txt index 328ae8308a4..162ab2c52e1 100644 --- a/ydb/core/tx/schemeshard/ut_split_merge/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_split_merge/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_split_merge PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_split_merge_reboots/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_split_merge_reboots/CMakeLists.darwin.txt index adfb3fc7b94..c193f778d2f 100644 --- a/ydb/core/tx/schemeshard/ut_split_merge_reboots/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_split_merge_reboots/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_split_merge_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_split_merge_reboots/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_split_merge_reboots/CMakeLists.linux.txt index 0d6581b727b..448e17cc1ef 100644 --- a/ydb/core/tx/schemeshard/ut_split_merge_reboots/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_split_merge_reboots/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_split_merge_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_stats/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_stats/CMakeLists.darwin.txt index f58a194f894..cb9622f83f7 100644 --- a/ydb/core/tx/schemeshard/ut_stats/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_stats/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_stats PUBLIC library-cpp-getopt cpp-regex-pcre ydb-core-cms - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers core-wrappers-ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_stats/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_stats/CMakeLists.linux.txt index 49492da49fc..58155ce1c95 100644 --- a/ydb/core/tx/schemeshard/ut_stats/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_stats/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_stats PUBLIC library-cpp-getopt cpp-regex-pcre ydb-core-cms - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers core-wrappers-ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_subdomain/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_subdomain/CMakeLists.darwin.txt index 91547ba8415..a096f56b155 100644 --- a/ydb/core/tx/schemeshard/ut_subdomain/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_subdomain/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_subdomain PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_subdomain/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_subdomain/CMakeLists.linux.txt index 80630bfe74a..c2dee0f1c3c 100644 --- a/ydb/core/tx/schemeshard/ut_subdomain/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_subdomain/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_subdomain PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_subdomain_reboots/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_subdomain_reboots/CMakeLists.darwin.txt index 563755e7206..c1bb342ae60 100644 --- a/ydb/core/tx/schemeshard/ut_subdomain_reboots/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_subdomain_reboots/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_subdomain_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_subdomain_reboots/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_subdomain_reboots/CMakeLists.linux.txt index 244150d99ce..04b8a5459cf 100644 --- a/ydb/core/tx/schemeshard/ut_subdomain_reboots/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_subdomain_reboots/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_subdomain_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_ttl/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_ttl/CMakeLists.darwin.txt index 217284f0418..90ad73fde6c 100644 --- a/ydb/core/tx/schemeshard/ut_ttl/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_ttl/CMakeLists.darwin.txt @@ -22,6 +22,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_ttl PUBLIC core-tx-schemeshard kqp-ut-common tx-schemeshard-ut_helpers + yql-sql-pg_dummy ) target_link_options(ydb-core-tx-schemeshard-ut_ttl PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/tx/schemeshard/ut_ttl/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_ttl/CMakeLists.linux.txt index 8fd121e25ef..71d930fc250 100644 --- a/ydb/core/tx/schemeshard/ut_ttl/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_ttl/CMakeLists.linux.txt @@ -24,6 +24,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_ttl PUBLIC core-tx-schemeshard kqp-ut-common tx-schemeshard-ut_helpers + yql-sql-pg_dummy ) target_link_options(ydb-core-tx-schemeshard-ut_ttl PRIVATE -ldl diff --git a/ydb/core/tx/schemeshard/ut_user_attributes/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_user_attributes/CMakeLists.darwin.txt index 1fa7238cd43..bd700aa60b2 100644 --- a/ydb/core/tx/schemeshard/ut_user_attributes/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_user_attributes/CMakeLists.darwin.txt @@ -31,7 +31,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_user_attributes PUBLIC ydb-core-scheme ydb-core-tablet ydb-core-tablet_flat - ydb-core-testlib + core-testlib-default ydb-core-tx core-tx-datashard tx-schemeshard-ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_user_attributes/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_user_attributes/CMakeLists.linux.txt index 4618c2059c5..db3461d468d 100644 --- a/ydb/core/tx/schemeshard/ut_user_attributes/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_user_attributes/CMakeLists.linux.txt @@ -33,7 +33,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_user_attributes PUBLIC ydb-core-scheme ydb-core-tablet ydb-core-tablet_flat - ydb-core-testlib + core-testlib-default ydb-core-tx core-tx-datashard tx-schemeshard-ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_user_attributes_reboots/CMakeLists.darwin.txt b/ydb/core/tx/schemeshard/ut_user_attributes_reboots/CMakeLists.darwin.txt index 32f2b4de33a..f65860e3c67 100644 --- a/ydb/core/tx/schemeshard/ut_user_attributes_reboots/CMakeLists.darwin.txt +++ b/ydb/core/tx/schemeshard/ut_user_attributes_reboots/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(core-tx-schemeshard-ut_user_attributes_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/schemeshard/ut_user_attributes_reboots/CMakeLists.linux.txt b/ydb/core/tx/schemeshard/ut_user_attributes_reboots/CMakeLists.linux.txt index f7cfeca9f79..11a2a778e80 100644 --- a/ydb/core/tx/schemeshard/ut_user_attributes_reboots/CMakeLists.linux.txt +++ b/ydb/core/tx/schemeshard/ut_user_attributes_reboots/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(core-tx-schemeshard-ut_user_attributes_reboots PUBLIC library-cpp-getopt cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy diff --git a/ydb/core/tx/sequenceproxy/ut/CMakeLists.darwin.txt b/ydb/core/tx/sequenceproxy/ut/CMakeLists.darwin.txt index 42dee28e6be..65f46fedb49 100644 --- a/ydb/core/tx/sequenceproxy/ut/CMakeLists.darwin.txt +++ b/ydb/core/tx/sequenceproxy/ut/CMakeLists.darwin.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-core-tx-sequenceproxy-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main core-tx-sequenceproxy - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-tx-sequenceproxy-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/tx/sequenceproxy/ut/CMakeLists.linux.txt b/ydb/core/tx/sequenceproxy/ut/CMakeLists.linux.txt index d3fbb0f4e1a..c3d882d80b2 100644 --- a/ydb/core/tx/sequenceproxy/ut/CMakeLists.linux.txt +++ b/ydb/core/tx/sequenceproxy/ut/CMakeLists.linux.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-tx-sequenceproxy-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main core-tx-sequenceproxy - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-tx-sequenceproxy-ut PRIVATE -ldl diff --git a/ydb/core/tx/sequenceshard/ut/CMakeLists.darwin.txt b/ydb/core/tx/sequenceshard/ut/CMakeLists.darwin.txt index 85eeef122ca..be53bd62865 100644 --- a/ydb/core/tx/sequenceshard/ut/CMakeLists.darwin.txt +++ b/ydb/core/tx/sequenceshard/ut/CMakeLists.darwin.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-core-tx-sequenceshard-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main core-tx-sequenceshard - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-tx-sequenceshard-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/tx/sequenceshard/ut/CMakeLists.linux.txt b/ydb/core/tx/sequenceshard/ut/CMakeLists.linux.txt index 0cc490d5cf2..ca5ddcaf19d 100644 --- a/ydb/core/tx/sequenceshard/ut/CMakeLists.linux.txt +++ b/ydb/core/tx/sequenceshard/ut/CMakeLists.linux.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-tx-sequenceshard-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main core-tx-sequenceshard - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-tx-sequenceshard-ut PRIVATE -ldl diff --git a/ydb/core/tx/time_cast/ut/CMakeLists.darwin.txt b/ydb/core/tx/time_cast/ut/CMakeLists.darwin.txt index 0dee18c7008..cffbcedb4d3 100644 --- a/ydb/core/tx/time_cast/ut/CMakeLists.darwin.txt +++ b/ydb/core/tx/time_cast/ut/CMakeLists.darwin.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-core-tx-time_cast-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main core-tx-time_cast - ydb-core-testlib + core-testlib-default ydb-core-tx ) target_link_options(ydb-core-tx-time_cast-ut PRIVATE diff --git a/ydb/core/tx/time_cast/ut/CMakeLists.linux.txt b/ydb/core/tx/time_cast/ut/CMakeLists.linux.txt index 7dcf54b6f11..1c370072079 100644 --- a/ydb/core/tx/time_cast/ut/CMakeLists.linux.txt +++ b/ydb/core/tx/time_cast/ut/CMakeLists.linux.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-tx-time_cast-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main core-tx-time_cast - ydb-core-testlib + core-testlib-default ydb-core-tx ) target_link_options(ydb-core-tx-time_cast-ut PRIVATE diff --git a/ydb/core/tx/tx_allocator/ut/CMakeLists.darwin.txt b/ydb/core/tx/tx_allocator/ut/CMakeLists.darwin.txt index 61b1088b737..ebfd6197df3 100644 --- a/ydb/core/tx/tx_allocator/ut/CMakeLists.darwin.txt +++ b/ydb/core/tx/tx_allocator/ut/CMakeLists.darwin.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-tx-tx_allocator-ut PUBLIC core-tx-tx_allocator cpp-testing-unittest ydb-core-mind - ydb-core-testlib + core-testlib-default ydb-core-tx ) target_link_options(ydb-core-tx-tx_allocator-ut PRIVATE diff --git a/ydb/core/tx/tx_allocator/ut/CMakeLists.linux.txt b/ydb/core/tx/tx_allocator/ut/CMakeLists.linux.txt index 6c798b1371c..2c3d733f818 100644 --- a/ydb/core/tx/tx_allocator/ut/CMakeLists.linux.txt +++ b/ydb/core/tx/tx_allocator/ut/CMakeLists.linux.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-tx_allocator-ut PUBLIC core-tx-tx_allocator cpp-testing-unittest ydb-core-mind - ydb-core-testlib + core-testlib-default ydb-core-tx ) target_link_options(ydb-core-tx-tx_allocator-ut PRIVATE diff --git a/ydb/core/tx/tx_allocator_client/ut/CMakeLists.darwin.txt b/ydb/core/tx/tx_allocator_client/ut/CMakeLists.darwin.txt index 99336186d81..9e0023845f3 100644 --- a/ydb/core/tx/tx_allocator_client/ut/CMakeLists.darwin.txt +++ b/ydb/core/tx/tx_allocator_client/ut/CMakeLists.darwin.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-tx-tx_allocator_client-ut PUBLIC core-tx-tx_allocator_client cpp-testing-unittest ydb-core-mind - ydb-core-testlib + core-testlib-default ydb-core-tx ) target_link_options(ydb-core-tx-tx_allocator_client-ut PRIVATE diff --git a/ydb/core/tx/tx_allocator_client/ut/CMakeLists.linux.txt b/ydb/core/tx/tx_allocator_client/ut/CMakeLists.linux.txt index c2f69a88c44..127aa0b3052 100644 --- a/ydb/core/tx/tx_allocator_client/ut/CMakeLists.linux.txt +++ b/ydb/core/tx/tx_allocator_client/ut/CMakeLists.linux.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-tx_allocator_client-ut PUBLIC core-tx-tx_allocator_client cpp-testing-unittest ydb-core-mind - ydb-core-testlib + core-testlib-default ydb-core-tx ) target_link_options(ydb-core-tx-tx_allocator_client-ut PRIVATE diff --git a/ydb/core/tx/tx_proxy/datareq.cpp b/ydb/core/tx/tx_proxy/datareq.cpp index cd37e9c42a0..41fe51828ed 100644 --- a/ydb/core/tx/tx_proxy/datareq.cpp +++ b/ydb/core/tx/tx_proxy/datareq.cpp @@ -19,6 +19,7 @@ #include <ydb/core/engine/mkql_engine_flat.h> #include <ydb/core/engine/mkql_proto.h> #include <ydb/core/scheme/scheme_types_defs.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/base/row_version.h> #include <ydb/library/yql/minikql/mkql_type_ops.h> @@ -91,9 +92,9 @@ class TKeySpace { public: TKeySpace(); - void Initialize(bool ordered, TConstArrayRef<NScheme::TTypeId> keyTypes, const TTableRange &range); + void Initialize(bool ordered, TConstArrayRef<NScheme::TTypeInfo> keyTypes, const TTableRange &range); - const TVector<NScheme::TTypeId> &GetKeyTypes() { return KeyTypes; } + const TVector<NScheme::TTypeInfo> &GetKeyTypes() { return KeyTypes; } void AddRange(const NKikimrTx::TKeyRange &range, ui64 shard); bool IsFull() const; @@ -126,7 +127,7 @@ private: void TryToMergeRange(TRanges::iterator it); bool TryToMergeWithPrev(TRanges::iterator it); - TVector<NScheme::TTypeId> KeyTypes; + TVector<NScheme::TTypeInfo> KeyTypes; TSerializedTableRange SpaceRange; TRanges Ranges; bool OrderedQueue; @@ -173,7 +174,7 @@ struct TReadTableRequest : public TThrRefBase { , RequestVersion(tx.HasApiVersion() ? tx.GetApiVersion() : (ui32)NKikimrTxUserProxy::TReadTableTransaction::UNSPECIFIED) { for (auto &col : tx.GetColumns()) { - Columns.emplace_back(col, 0, 0); + Columns.emplace_back(col, 0, NScheme::TTypeInfo(0)); } if (tx.HasSnapshotStep() && tx.HasSnapshotTxId()) { @@ -444,7 +445,7 @@ private: void ProcessStreamClearance(bool cleared, const TActorContext &ctx); bool ParseRangeKey(const NKikimrMiniKQL::TParams &proto, - TConstArrayRef<NScheme::TTypeId> keyType, + TConstArrayRef<NScheme::TTypeInfo> keyType, TSerializedCellVec &buf, EParseRangeKeyExp exp); @@ -584,7 +585,7 @@ TKeySpace::TKeySpace() } void TKeySpace::Initialize(bool ordered, - TConstArrayRef<NScheme::TTypeId> keyTypes, + TConstArrayRef<NScheme::TTypeInfo> keyTypes, const TTableRange &range) { SpaceRange.From.Parse(TSerializedCellVec::Serialize(range.From)); @@ -1161,7 +1162,11 @@ void TDataReq::ProcessReadTableResolve(NSchemeCache::TSchemeCacheRequest *cacheR auto &c = *tx.AddColumns(); c.SetId(col.Id); c.SetName(col.Name); - c.SetTypeId(col.PType); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(col.PType); + c.SetTypeId(columnType.TypeId); + if (columnType.TypeInfo) { + *c.MutableTypeInfo() = *columnType.TypeInfo; + } } auto &range = *tx.MutableRange(); ReadTableRequest->KeySpace.GetSpace().Serialize(range); @@ -1499,7 +1504,7 @@ void TDataReq::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr &ev, return Die(ctx); } - TVector<NScheme::TTypeId> keyColumnTypes(res.Columns.size()); + TVector<NScheme::TTypeInfo> keyColumnTypes(res.Columns.size()); TVector<TKeyDesc::TColumnOp> columns(res.Columns.size()); size_t keySize = 0; size_t no = 0; @@ -1540,7 +1545,7 @@ void TDataReq::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr &ev, } // Parse range. - TConstArrayRef<NScheme::TTypeId> keyTypes(keyColumnTypes.data(), keySize); + TConstArrayRef<NScheme::TTypeInfo> keyTypes(keyColumnTypes.data(), keySize); // Fix KeyRanges bool fromInclusive = ReadTableRequest->Range.GetFromInclusive(); EParseRangeKeyExp fromExpand = EParseRangeKeyExp::TO_NULL; @@ -3013,7 +3018,7 @@ void TDataReq::ProcessStreamClearance(bool cleared, const TActorContext &ctx) } bool TDataReq::ParseRangeKey(const NKikimrMiniKQL::TParams &proto, - TConstArrayRef<NScheme::TTypeId> keyType, + TConstArrayRef<NScheme::TTypeInfo> keyType, TSerializedCellVec &buf, EParseRangeKeyExp exp) { diff --git a/ydb/core/tx/tx_proxy/describe.cpp b/ydb/core/tx/tx_proxy/describe.cpp index deb38ca2b7d..e718ac25204 100644 --- a/ydb/core/tx/tx_proxy/describe.cpp +++ b/ydb/core/tx/tx_proxy/describe.cpp @@ -3,6 +3,7 @@ #include <ydb/core/tx/scheme_cache/scheme_cache.h> #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/scheme/scheme_types_defs.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/base/tablet_pipecache.h> #include <ydb/core/base/appdata.h> #include <ydb/core/sys_view/common/schema.h> @@ -102,8 +103,13 @@ class TDescribeReq : public TActor<TDescribeReq> { for (const auto& [id, column] : entry.Columns) { auto* col = table->AddColumns(); col->SetName(column.Name); - col->SetType(AppData(ctx)->TypeRegistry->GetTypeName(column.PType)); - col->SetTypeId(column.PType); + // TODO: support pg types (name) + col->SetType(AppData(ctx)->TypeRegistry->GetTypeName(column.PType.GetTypeId())); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.PType); + col->SetTypeId(columnType.TypeId); + if (columnType.TypeInfo) { + *col->MutableTypeInfo() = *columnType.TypeInfo; + } col->SetId(id); if (column.KeyOrder >= 0) { Y_VERIFY((size_t)column.KeyOrder < keyColumnIds.size()); diff --git a/ydb/core/tx/tx_proxy/read_table_impl.cpp b/ydb/core/tx/tx_proxy/read_table_impl.cpp index f2d75397878..7cb9a0e0a6e 100644 --- a/ydb/core/tx/tx_proxy/read_table_impl.cpp +++ b/ydb/core/tx/tx_proxy/read_table_impl.cpp @@ -8,6 +8,7 @@ #include <ydb/core/base/path.h> #include <ydb/core/base/tablet_pipecache.h> #include <ydb/core/scheme/scheme_borders.h> +#include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/engine/mkql_proto.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> @@ -52,7 +53,7 @@ namespace { * Note that range begin/end are treated as key prefixes if they are incomplete. */ bool RangeEndBeginHasIntersection( - TConstArrayRef<NScheme::TTypeId> keyTypes, + TConstArrayRef<NScheme::TTypeInfo> keyTypes, TConstArrayRef<TCell> rangeEnd, bool rangeEndInclusive, TConstArrayRef<TCell> rangeBegin, bool rangeBeginInclusive) { @@ -69,7 +70,7 @@ namespace { } int CompareRangeEnds( - TConstArrayRef<NScheme::TTypeId> keyTypes, + TConstArrayRef<NScheme::TTypeInfo> keyTypes, TConstArrayRef<TCell> left, bool leftInclusive, TConstArrayRef<TCell> right, bool rightInclusive) { @@ -88,7 +89,7 @@ namespace { bool ParseRangeKey( const NKikimrMiniKQL::TParams& proto, - TConstArrayRef<NScheme::TTypeId> keyTypes, + TConstArrayRef<NScheme::TTypeInfo> keyTypes, TSerializedCellVec& buf, EParseRangeKeyExp exp, TVector<TString>& unresolvedKeys) @@ -527,7 +528,7 @@ private: return ReplyAndDie(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ResolveError, NKikimrIssues::TStatusIds::SCHEME_ERROR, ctx); } - TVector<NScheme::TTypeId> keyTypes(res.Columns.size()); + TVector<NScheme::TTypeInfo> keyTypes(res.Columns.size()); TVector<TKeyDesc::TColumnOp> columns(res.Columns.size()); { size_t no = 0; @@ -1432,7 +1433,11 @@ private: auto &c = *tx.AddColumns(); c.SetId(col.Id); c.SetName(col.Name); - c.SetTypeId(col.PType); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(col.PType); + c.SetTypeId(columnType.TypeId); + if (columnType.TypeInfo) { + *c.MutableTypeInfo() = *columnType.TypeInfo; + } } auto& txRange = *tx.MutableRange(); diff --git a/ydb/core/tx/tx_proxy/resolvereq.cpp b/ydb/core/tx/tx_proxy/resolvereq.cpp index d01f795e495..6ad8cb64448 100644 --- a/ydb/core/tx/tx_proxy/resolvereq.cpp +++ b/ydb/core/tx/tx_proxy/resolvereq.cpp @@ -19,7 +19,7 @@ namespace { bool ParseRangeKey( const NKikimrMiniKQL::TParams& proto, - TConstArrayRef<NScheme::TTypeId> keyTypes, + TConstArrayRef<NScheme::TTypeInfo> keyTypes, TSerializedCellVec& buf, EParseRangeKeyExp exp, TVector<TString>& unresolvedKeys) @@ -182,7 +182,7 @@ namespace { table.TableId = entry.TableId; table.IsColumnTable = (entry.Kind == NSchemeCache::TSchemeCacheNavigate::KindColumnTable); - TVector<NScheme::TTypeId> keyColumnTypes(entry.Columns.size()); + TVector<NScheme::TTypeInfo> keyColumnTypes(entry.Columns.size()); TVector<TKeyDesc::TColumnOp> columns(entry.Columns.size()); size_t keySize = 0; size_t no = 0; @@ -202,7 +202,7 @@ namespace { } // Parse range. - TConstArrayRef<NScheme::TTypeId> keyTypes(keyColumnTypes.data(), keySize); + TConstArrayRef<NScheme::TTypeInfo> keyTypes(keyColumnTypes.data(), keySize); const bool fromInclusive = table.KeyRange.GetFromInclusive(); const bool toInclusive = table.KeyRange.GetToInclusive(); diff --git a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h index 9fa28104477..83c99145215 100644 --- a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h +++ b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h @@ -12,6 +12,7 @@ #include <ydb/core/base/kikimr_issue.h> #include <ydb/core/base/path.h> #include <ydb/core/scheme/scheme_tablecell.h> +#include <ydb/core/scheme/scheme_type_info.h> #include <ydb/core/tx/datashard/datashard.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> @@ -114,8 +115,8 @@ private: std::shared_ptr<NSchemeCache::TSchemeCacheNavigate> ResolveNamesResult; TSerializedCellVec MinKey; TSerializedCellVec MaxKey; - TVector<NScheme::TTypeId> KeyColumnTypes; - TVector<NScheme::TTypeId> ValueColumnTypes; + TVector<NScheme::TTypeInfo> KeyColumnTypes; + TVector<NScheme::TTypeInfo> ValueColumnTypes; NSchemeCache::TSchemeCacheNavigate::EKind TableKind = NSchemeCache::TSchemeCacheNavigate::KindUnknown; THashSet<TTabletId> ShardRepliesLeft; Ydb::StatusIds::StatusCode Status; @@ -136,7 +137,7 @@ protected: ui32 ColId; TString ColName; ui32 PositionInStruct; - NScheme::TTypeId Type; + NScheme::TTypeInfo Type; bool NotNull = false; }; TVector<TString> KeyColumnNames; @@ -145,10 +146,10 @@ protected: TVector<TFieldDescription> ValueColumnPositions; // Additional schema info (for OLAP dst or source format) - TVector<std::pair<TString, NScheme::TTypeId>> SrcColumns; // source columns in CSV could have any order - TVector<std::pair<TString, NScheme::TTypeId>> YdbSchema; + TVector<std::pair<TString, NScheme::TTypeInfo>> SrcColumns; // source columns in CSV could have any order + TVector<std::pair<TString, NScheme::TTypeInfo>> YdbSchema; THashMap<ui32, size_t> Id2Position; // columnId -> its position in YdbSchema - THashMap<TString, NScheme::TTypeId> ColumnsToConvert; + THashMap<TString, NScheme::TTypeInfo> ColumnsToConvert; bool WriteToTableShadow = false; bool AllowWriteToPrivateTable = false; @@ -271,7 +272,7 @@ private: } } - static bool SameDstType(NScheme::TTypeId type1, NScheme::TTypeId type2, bool allowConvert) { + static bool SameDstType(NScheme::TTypeInfo type1, NScheme::TTypeInfo type2, bool allowConvert) { bool res = (type1 == type2); if (!res && allowConvert) { res = NArrow::GetArrowType(type1)->id() == NArrow::GetArrowType(type2)->id(); @@ -333,7 +334,9 @@ private: } else if (reqColumns.empty()) { for (auto& [name, type] : SrcColumns) { Ydb::Type ydbType; - ydbType.set_type_id((Ydb::Type::PrimitiveTypeId)type); + // TODO: support pg types + Y_VERIFY(type.GetTypeId() != NScheme::NTypeIds::Pg); + ydbType.set_type_id((Ydb::Type::PrimitiveTypeId)type.GetTypeId()); reqColumns.emplace_back(name, std::move(ydbType)); } } @@ -351,7 +354,7 @@ private: const auto& typeInProto = reqColumns[pos].second; if (typeInProto.type_id()) { - NScheme::TTypeId typeInRequest = typeInProto.type_id(); + auto typeInRequest = NScheme::TTypeInfo(typeInProto.type_id()); bool ok = SameDstType(typeInRequest, ci.PType, GetSourceType() != EUploadSource::ProtoValues); if (!ok) { errorMessage = Sprintf("Type mismatch for column %s: expected %s, got %s", @@ -359,7 +362,7 @@ private: NScheme::TypeName(typeInRequest)); return false; } - } else if (typeInProto.has_decimal_type() && ci.PType == NScheme::NTypeIds::Decimal) { + } else if (typeInProto.has_decimal_type() && ci.PType.GetTypeId() == NScheme::NTypeIds::Decimal) { int precision = typeInProto.decimal_type().precision(); int scale = typeInProto.decimal_type().scale(); if (precision != NScheme::DECIMAL_PRECISION || scale != NScheme::DECIMAL_SCALE) { @@ -370,8 +373,24 @@ private: return false; } + } else if (typeInProto.has_pg_type()) { + auto pgTypeId = typeInProto.pg_type().oid(); + auto* typeDesc = NPg::TypeDescFromPgTypeId(pgTypeId); + if (!typeDesc) { + errorMessage = Sprintf("Unknown pg type for column %s: %u", + name.c_str(), pgTypeId); + return false; + } + auto typeInRequest = NScheme::TTypeInfo(NScheme::NTypeIds::Pg, typeDesc); + bool ok = SameDstType(typeInRequest, ci.PType, false); + if (!ok) { + errorMessage = Sprintf("Type mismatch for column %s: expected %s, got %s", + name.c_str(), NScheme::TypeName(ci.PType), + NScheme::TypeName(typeInRequest)); + return false; + } } else { - errorMessage = Sprintf("Unexected type for column %s: expected %s", + errorMessage = Sprintf("Unexpected type for column %s: expected %s", name.c_str(), NScheme::TypeName(ci.PType)); return false; } @@ -381,13 +400,12 @@ private: notNullColumnsLeft.erase(ci.Name); } - NScheme::TTypeId typeId = (NScheme::TTypeId)ci.PType; if (ci.KeyOrder != -1) { - KeyColumnPositions[ci.KeyOrder] = TFieldDescription{ci.Id, ci.Name, (ui32)pos, typeId, notNull}; + KeyColumnPositions[ci.KeyOrder] = TFieldDescription{ci.Id, ci.Name, (ui32)pos, ci.PType, notNull}; keyColumnsLeft.erase(ci.Name); KeyColumnNames[ci.KeyOrder] = ci.Name; } else { - ValueColumnPositions.emplace_back(TFieldDescription{ci.Id, ci.Name, (ui32)pos, typeId, notNull}); + ValueColumnPositions.emplace_back(TFieldDescription{ci.Id, ci.Name, (ui32)pos, ci.PType, notNull}); ValueColumnNames.emplace_back(ci.Name); ValueColumnTypes.emplace_back(ci.PType); } diff --git a/ydb/core/tx/tx_proxy/ut_base_tenant/CMakeLists.darwin.txt b/ydb/core/tx/tx_proxy/ut_base_tenant/CMakeLists.darwin.txt index 0812000707e..353f60ea638 100644 --- a/ydb/core/tx/tx_proxy/ut_base_tenant/CMakeLists.darwin.txt +++ b/ydb/core/tx/tx_proxy/ut_base_tenant/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-tx_proxy-ut_base_tenant PUBLIC library-cpp-getopt library-cpp-svnversion cpp-testing-unittest - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy ) diff --git a/ydb/core/tx/tx_proxy/ut_base_tenant/CMakeLists.linux.txt b/ydb/core/tx/tx_proxy/ut_base_tenant/CMakeLists.linux.txt index 0e44e65e5de..5a882b33700 100644 --- a/ydb/core/tx/tx_proxy/ut_base_tenant/CMakeLists.linux.txt +++ b/ydb/core/tx/tx_proxy/ut_base_tenant/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-tx_proxy-ut_base_tenant PUBLIC library-cpp-getopt library-cpp-svnversion cpp-testing-unittest - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy ) diff --git a/ydb/core/tx/tx_proxy/ut_encrypted_storage/CMakeLists.darwin.txt b/ydb/core/tx/tx_proxy/ut_encrypted_storage/CMakeLists.darwin.txt index 2daa4feed9d..1a76230cb80 100644 --- a/ydb/core/tx/tx_proxy/ut_encrypted_storage/CMakeLists.darwin.txt +++ b/ydb/core/tx/tx_proxy/ut_encrypted_storage/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-tx_proxy-ut_encrypted_storage PUBLIC library-cpp-getopt library-cpp-svnversion cpp-testing-unittest - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy ) diff --git a/ydb/core/tx/tx_proxy/ut_encrypted_storage/CMakeLists.linux.txt b/ydb/core/tx/tx_proxy/ut_encrypted_storage/CMakeLists.linux.txt index e05854e5088..b565aea5fa4 100644 --- a/ydb/core/tx/tx_proxy/ut_encrypted_storage/CMakeLists.linux.txt +++ b/ydb/core/tx/tx_proxy/ut_encrypted_storage/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-tx_proxy-ut_encrypted_storage PUBLIC library-cpp-getopt library-cpp-svnversion cpp-testing-unittest - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy ) diff --git a/ydb/core/tx/tx_proxy/ut_ext_tenant/CMakeLists.darwin.txt b/ydb/core/tx/tx_proxy/ut_ext_tenant/CMakeLists.darwin.txt index 8267e24b875..55adb49d8ff 100644 --- a/ydb/core/tx/tx_proxy/ut_ext_tenant/CMakeLists.darwin.txt +++ b/ydb/core/tx/tx_proxy/ut_ext_tenant/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-tx_proxy-ut_ext_tenant PUBLIC library-cpp-getopt library-cpp-svnversion cpp-testing-unittest - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy ) diff --git a/ydb/core/tx/tx_proxy/ut_ext_tenant/CMakeLists.linux.txt b/ydb/core/tx/tx_proxy/ut_ext_tenant/CMakeLists.linux.txt index c56589fbd09..551794983f8 100644 --- a/ydb/core/tx/tx_proxy/ut_ext_tenant/CMakeLists.linux.txt +++ b/ydb/core/tx/tx_proxy/ut_ext_tenant/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-tx_proxy-ut_ext_tenant PUBLIC library-cpp-getopt library-cpp-svnversion cpp-testing-unittest - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy ) diff --git a/ydb/core/tx/tx_proxy/ut_storage_tenant/CMakeLists.darwin.txt b/ydb/core/tx/tx_proxy/ut_storage_tenant/CMakeLists.darwin.txt index 64ca795a624..413682b3073 100644 --- a/ydb/core/tx/tx_proxy/ut_storage_tenant/CMakeLists.darwin.txt +++ b/ydb/core/tx/tx_proxy/ut_storage_tenant/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-tx_proxy-ut_storage_tenant PUBLIC library-cpp-svnversion cpp-testing-unittest ydb-core-blobstorage - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy ) diff --git a/ydb/core/tx/tx_proxy/ut_storage_tenant/CMakeLists.linux.txt b/ydb/core/tx/tx_proxy/ut_storage_tenant/CMakeLists.linux.txt index f02b120c779..fef0f8a620b 100644 --- a/ydb/core/tx/tx_proxy/ut_storage_tenant/CMakeLists.linux.txt +++ b/ydb/core/tx/tx_proxy/ut_storage_tenant/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-tx-tx_proxy-ut_storage_tenant PUBLIC library-cpp-svnversion cpp-testing-unittest ydb-core-blobstorage - ydb-core-testlib + core-testlib-default ydb-core-tx udf-service-exception_policy ) diff --git a/ydb/core/viewer/ut/CMakeLists.darwin.txt b/ydb/core/viewer/ut/CMakeLists.darwin.txt index b7c4189ce10..d5265c57428 100644 --- a/ydb/core/viewer/ut/CMakeLists.darwin.txt +++ b/ydb/core/viewer/ut/CMakeLists.darwin.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-core-viewer-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main ydb-core-viewer - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-viewer-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/viewer/ut/CMakeLists.linux.txt b/ydb/core/viewer/ut/CMakeLists.linux.txt index 6ef556bbe63..ce2706d4687 100644 --- a/ydb/core/viewer/ut/CMakeLists.linux.txt +++ b/ydb/core/viewer/ut/CMakeLists.linux.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-viewer-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main ydb-core-viewer - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-viewer-ut PRIVATE -ldl diff --git a/ydb/core/wrappers/ut/CMakeLists.darwin.txt b/ydb/core/wrappers/ut/CMakeLists.darwin.txt index 0a645d66c12..0272b23348a 100644 --- a/ydb/core/wrappers/ut/CMakeLists.darwin.txt +++ b/ydb/core/wrappers/ut/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-wrappers-ut PUBLIC cpp-digest-md5 cpp-testing-unittest ydb-core-protos - core-testlib-basics + testlib-basics-default core-wrappers-ut_helpers ) target_link_options(ydb-core-wrappers-ut PRIVATE diff --git a/ydb/core/wrappers/ut/CMakeLists.linux.txt b/ydb/core/wrappers/ut/CMakeLists.linux.txt index f6c7a13d97f..58a766a89d5 100644 --- a/ydb/core/wrappers/ut/CMakeLists.linux.txt +++ b/ydb/core/wrappers/ut/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-core-wrappers-ut PUBLIC cpp-digest-md5 cpp-testing-unittest ydb-core-protos - core-testlib-basics + testlib-basics-default core-wrappers-ut_helpers ) target_link_options(ydb-core-wrappers-ut PRIVATE diff --git a/ydb/core/ydb_convert/table_description.cpp b/ydb/core/ydb_convert/table_description.cpp index 16ce5355732..dd2e68ef6eb 100644 --- a/ydb/core/ydb_convert/table_description.cpp +++ b/ydb/core/ydb_convert/table_description.cpp @@ -123,8 +123,8 @@ void FillColumnDescription(Ydb::Table::CreateTableRequest& out, FillColumnDescriptionImpl(out, splitKeyType, in); } -bool ExtractColumnTypeId(ui32& outTypeId, const Ydb::Type& inType, Ydb::StatusIds::StatusCode& status, TString& error) { - ui32 typeId; +bool ExtractColumnTypeInfo(NScheme::TTypeInfo& outTypeInfo, const Ydb::Type& inType, Ydb::StatusIds::StatusCode& status, TString& error) { + ui32 typeId = 0; auto itemType = inType.has_optional_type() ? inType.optional_type().item() : inType; switch (itemType.type_case()) { case Ydb::Type::kTypeId: @@ -150,6 +150,17 @@ bool ExtractColumnTypeId(ui32& outTypeId, const Ydb::Type& inType, Ydb::StatusId typeId = NYql::NProto::TypeIds::Decimal; break; } + case Ydb::Type::kPgType: { + ui32 pgTypeId = itemType.pg_type().oid(); + auto* desc = NPg::TypeDescFromPgTypeId(pgTypeId); + if (!desc) { + status = Ydb::StatusIds::BAD_REQUEST; + error = TStringBuilder() << "Invalid PG typeId: " << pgTypeId; + return false; + } + outTypeInfo = NScheme::TTypeInfo(NScheme::NTypeIds::Pg, desc); + return true; + } default: { status = Ydb::StatusIds::BAD_REQUEST; @@ -164,7 +175,7 @@ bool ExtractColumnTypeId(ui32& outTypeId, const Ydb::Type& inType, Ydb::StatusId return false; } - outTypeId = typeId; + outTypeInfo = NScheme::TTypeInfo(typeId); return true; } @@ -184,11 +195,11 @@ bool FillColumnDescription(NKikimrSchemeOp::TTableDescription& out, cd->SetNotNull(true); } - ui32 typeId; - if (!ExtractColumnTypeId(typeId, column.type(), status, error)) { + NScheme::TTypeInfo typeInfo; + if (!ExtractColumnTypeInfo(typeInfo, column.type(), status, error)) { return false; } - cd->SetType(NYql::NProto::TypeIds_Name(NYql::NProto::TypeIds(typeId))); + cd->SetType(NScheme::TypeName(typeInfo)); if (!column.family().empty()) { cd->SetFamilyName(column.family()); diff --git a/ydb/core/ydb_convert/table_description.h b/ydb/core/ydb_convert/table_description.h index 9dc023288aa..198171f57c5 100644 --- a/ydb/core/ydb_convert/table_description.h +++ b/ydb/core/ydb_convert/table_description.h @@ -4,6 +4,7 @@ #include <ydb/library/mkql_proto/protos/minikql.pb.h> #include <ydb/core/protos/flat_tx_scheme.pb.h> +#include <ydb/core/scheme/scheme_type_info.h> #include <ydb/public/api/protos/ydb_table.pb.h> namespace NKikimr { @@ -16,7 +17,7 @@ void FillColumnDescription(Ydb::Table::CreateTableRequest& out, // in bool FillColumnDescription(NKikimrSchemeOp::TTableDescription& out, const google::protobuf::RepeatedPtrField<Ydb::Table::ColumnMeta>& in, Ydb::StatusIds::StatusCode& status, TString& error); -bool ExtractColumnTypeId(ui32& outTypeId, const Ydb::Type& inType, Ydb::StatusIds::StatusCode& status, TString& error); +bool ExtractColumnTypeInfo(NScheme::TTypeInfo& outTypeInfo, const Ydb::Type& inType, Ydb::StatusIds::StatusCode& status, TString& error); // out void FillTableBoundary(Ydb::Table::DescribeTableResult& out, diff --git a/ydb/core/ydb_convert/ut/CMakeLists.darwin.txt b/ydb/core/ydb_convert/ut/CMakeLists.darwin.txt index 6302119a471..66fbb3e2ca7 100644 --- a/ydb/core/ydb_convert/ut/CMakeLists.darwin.txt +++ b/ydb/core/ydb_convert/ut/CMakeLists.darwin.txt @@ -18,7 +18,7 @@ target_link_libraries(ydb-core-ydb_convert-ut PUBLIC cpp-testing-unittest_main ydb-core-ydb_convert cpp-testing-unittest - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-ydb_convert-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/ydb_convert/ut/CMakeLists.linux.txt b/ydb/core/ydb_convert/ut/CMakeLists.linux.txt index b617a98a7ae..c3930caf1fd 100644 --- a/ydb/core/ydb_convert/ut/CMakeLists.linux.txt +++ b/ydb/core/ydb_convert/ut/CMakeLists.linux.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-core-ydb_convert-ut PUBLIC cpp-testing-unittest_main ydb-core-ydb_convert cpp-testing-unittest - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-core-ydb_convert-ut PRIVATE -ldl diff --git a/ydb/core/ymq/actor/ut/CMakeLists.darwin.txt b/ydb/core/ymq/actor/ut/CMakeLists.darwin.txt index 2875fa1e7e2..a761bd6907f 100644 --- a/ydb/core/ymq/actor/ut/CMakeLists.darwin.txt +++ b/ydb/core/ymq/actor/ut/CMakeLists.darwin.txt @@ -18,7 +18,7 @@ target_link_libraries(ydb-core-ymq-actor-ut PUBLIC cpp-testing-unittest_main contrib-libs-yaml-cpp core-mind-address_classification - ydb-core-testlib + core-testlib-default core-ymq-actor core-ymq-base core-ymq-http diff --git a/ydb/core/ymq/actor/ut/CMakeLists.linux.txt b/ydb/core/ymq/actor/ut/CMakeLists.linux.txt index ecbcfdfd97a..f95d6ba95ea 100644 --- a/ydb/core/ymq/actor/ut/CMakeLists.linux.txt +++ b/ydb/core/ymq/actor/ut/CMakeLists.linux.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-core-ymq-actor-ut PUBLIC cpp-testing-unittest_main contrib-libs-yaml-cpp core-mind-address_classification - ydb-core-testlib + core-testlib-default core-ymq-actor core-ymq-base core-ymq-http diff --git a/ydb/core/ymq/actor/yc_search_ut/CMakeLists.darwin.txt b/ydb/core/ymq/actor/yc_search_ut/CMakeLists.darwin.txt index 8097c63bd5b..22b87a34c89 100644 --- a/ydb/core/ymq/actor/yc_search_ut/CMakeLists.darwin.txt +++ b/ydb/core/ymq/actor/yc_search_ut/CMakeLists.darwin.txt @@ -16,7 +16,7 @@ target_link_libraries(ydb-core-ymq-actor-yc_search_ut PUBLIC yutil library-cpp-cpuid_check cpp-testing-unittest_main - ydb-core-testlib + core-testlib-default core-ymq-actor ) target_link_options(ydb-core-ymq-actor-yc_search_ut PRIVATE diff --git a/ydb/core/ymq/actor/yc_search_ut/CMakeLists.linux.txt b/ydb/core/ymq/actor/yc_search_ut/CMakeLists.linux.txt index d707140eaa8..4f2c1b631a7 100644 --- a/ydb/core/ymq/actor/yc_search_ut/CMakeLists.linux.txt +++ b/ydb/core/ymq/actor/yc_search_ut/CMakeLists.linux.txt @@ -18,7 +18,7 @@ target_link_libraries(ydb-core-ymq-actor-yc_search_ut PUBLIC libs-tcmalloc-no_percpu_cache library-cpp-cpuid_check cpp-testing-unittest_main - ydb-core-testlib + core-testlib-default core-ymq-actor ) target_link_options(ydb-core-ymq-actor-yc_search_ut PRIVATE diff --git a/ydb/core/yq/libs/checkpointing/ut/CMakeLists.darwin.txt b/ydb/core/yq/libs/checkpointing/ut/CMakeLists.darwin.txt index f809e1b0f76..2531d978771 100644 --- a/ydb/core/yq/libs/checkpointing/ut/CMakeLists.darwin.txt +++ b/ydb/core/yq/libs/checkpointing/ut/CMakeLists.darwin.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-yq-libs-checkpointing-ut PUBLIC yq-libs-checkpointing cpp-testing-unittest core-testlib-actors - core-testlib-basics + testlib-basics-default ) target_link_options(ydb-core-yq-libs-checkpointing-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/core/yq/libs/checkpointing/ut/CMakeLists.linux.txt b/ydb/core/yq/libs/checkpointing/ut/CMakeLists.linux.txt index f41a4b1ebfd..48aaf61320b 100644 --- a/ydb/core/yq/libs/checkpointing/ut/CMakeLists.linux.txt +++ b/ydb/core/yq/libs/checkpointing/ut/CMakeLists.linux.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-yq-libs-checkpointing-ut PUBLIC yq-libs-checkpointing cpp-testing-unittest core-testlib-actors - core-testlib-basics + testlib-basics-default ) target_link_options(ydb-core-yq-libs-checkpointing-ut PRIVATE -ldl diff --git a/ydb/core/yq/libs/control_plane_proxy/ut/CMakeLists.darwin.txt b/ydb/core/yq/libs/control_plane_proxy/ut/CMakeLists.darwin.txt index 410d65c8611..8ab7757626a 100644 --- a/ydb/core/yq/libs/control_plane_proxy/ut/CMakeLists.darwin.txt +++ b/ydb/core/yq/libs/control_plane_proxy/ut/CMakeLists.darwin.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-core-yq-libs-control_plane_proxy-ut PUBLIC yq-libs-control_plane_proxy cpp-testing-unittest ydb-core-base - ydb-core-testlib + core-testlib-default libs-actors-logging yq-libs-control_plane_storage yq-libs-test_connection diff --git a/ydb/core/yq/libs/control_plane_proxy/ut/CMakeLists.linux.txt b/ydb/core/yq/libs/control_plane_proxy/ut/CMakeLists.linux.txt index 3a0132a9260..c0d7a10def3 100644 --- a/ydb/core/yq/libs/control_plane_proxy/ut/CMakeLists.linux.txt +++ b/ydb/core/yq/libs/control_plane_proxy/ut/CMakeLists.linux.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-yq-libs-control_plane_proxy-ut PUBLIC yq-libs-control_plane_proxy cpp-testing-unittest ydb-core-base - ydb-core-testlib + core-testlib-default libs-actors-logging yq-libs-control_plane_storage yq-libs-test_connection diff --git a/ydb/core/yql_testlib/CMakeLists.txt b/ydb/core/yql_testlib/CMakeLists.txt index 86d11c2156b..4a617c586e0 100644 --- a/ydb/core/yql_testlib/CMakeLists.txt +++ b/ydb/core/yql_testlib/CMakeLists.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-yql_testlib PUBLIC ydb-core-keyvalue ydb-core-mind ydb-core-protos - ydb-core-testlib + core-testlib-default core-testlib-actors core-testlib-basics yql-core-facade diff --git a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp index 3ca55e206cf..9376ec2ac70 100644 --- a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp +++ b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp @@ -2903,3 +2903,264 @@ void get_type_io_data(Oid typid, } } // extern "C" + +namespace NKikimr::NPg { + +class TPgTypeDescriptor + : public NYql::NPg::TTypeDesc +{ +public: + TPgTypeDescriptor() + {} + + explicit TPgTypeDescriptor(const NYql::NPg::TTypeDesc& desc) + : NYql::NPg::TTypeDesc(desc) + { + // TODO: btarraycmp and hash_array for array types + if (CompareProcId) { + InitFunc(CompareProcId, &CompareProcInfo, 2, 2); + } + if (HashProcId) { + InitFunc(HashProcId, &HashProcInfo, 1, 1); + } + if (ReceiveFuncId) { + InitFunc(ReceiveFuncId, &ReceiveFuncInfo, 1, 3); + } + if (SendFuncId) { + InitFunc(SendFuncId, &SendFuncInfo, 1, 1); + } + } + + int Compare(const char* dataL, size_t sizeL, const char* dataR, size_t sizeR) const { + NMiniKQL::TScopedAlloc alloc; + NMiniKQL::TPAllocScope scope; + PG_TRY(); + { + Datum datumL, datumR; + if (PassByValue) { + datumL = ScalarDatumFromData(dataL, sizeL); + datumR = ScalarDatumFromData(dataR, sizeR); + } else { + datumL = Unpack(dataL, sizeL); + datumR = Unpack(dataR, sizeR); + } + + LOCAL_FCINFO(callInfo, 2); + Zero(*callInfo); + callInfo->flinfo = (FmgrInfo*)&CompareProcInfo; + callInfo->nargs = 2; + callInfo->fncollation = DEFAULT_COLLATION_OID; + callInfo->isnull = false; + callInfo->args[0] = { datumL, false }; + callInfo->args[1] = { datumR, false }; + + auto result = CompareProcInfo.fn_addr(callInfo); + Y_ENSURE(!callInfo->isnull); + if (!PassByValue) { + pfree((void*)datumL); + pfree((void*)datumR); + } + return DatumGetInt32(result); + } + PG_CATCH(); + { + // TODO + } + PG_END_TRY(); + return 0; + } + + ui64 Hash(const char* data, size_t size) const { + NMiniKQL::TScopedAlloc alloc; + NMiniKQL::TPAllocScope scope; + PG_TRY(); + { + Datum datum; + if (PassByValue) { + datum = ScalarDatumFromData(data, size); + } else { + datum = Unpack(data, size); + } + + LOCAL_FCINFO(callInfo, 1); + Zero(*callInfo); + callInfo->flinfo = (FmgrInfo*)&HashProcInfo; + callInfo->nargs = 1; + callInfo->fncollation = DEFAULT_COLLATION_OID; + callInfo->isnull = false; + callInfo->args[0] = { datum, false }; + + auto result = HashProcInfo.fn_addr(callInfo); + Y_ENSURE(!callInfo->isnull); + if (!PassByValue) { + pfree((void*)datum); + } + return DatumGetUInt32(result); + } + PG_CATCH(); + { + // TODO + } + PG_END_TRY(); + return 0; + } + +private: + inline Datum ScalarDatumFromData(const char* data, size_t size) const { + switch (TypeId) { + case BOOLOID: + Y_ENSURE(size == sizeof(bool)); + return BoolGetDatum(ReadUnaligned<bool>(data)); + case CHAROID: + Y_ENSURE(size == sizeof(char)); + return CharGetDatum(ReadUnaligned<char>(data)); + case INT2OID: + Y_ENSURE(size == sizeof(i16)); + return Int16GetDatum(ReadUnaligned<i16>(data)); + case INT4OID: + Y_ENSURE(size == sizeof(i32)); + return Int32GetDatum(ReadUnaligned<i32>(data)); + case INT8OID: + Y_ENSURE(size == sizeof(i64)); + return Int64GetDatum(ReadUnaligned<i64>(data)); + case FLOAT4OID: + Y_ENSURE(size == sizeof(float)); + return Float4GetDatum(ReadUnaligned<float>(data)); + case FLOAT8OID: + Y_ENSURE(size == sizeof(double)); + return Float8GetDatum(ReadUnaligned<double>(data)); + default: { + Y_ENSURE(size <= sizeof(ui64)); + ui64 val = 0; + std::memcpy(&val, data, size); + return (Datum)val; + } + } + } + + Datum Unpack(const char* data, size_t size) const { + StringInfoData stringInfo; + stringInfo.data = (char*)data; + stringInfo.len = size; + stringInfo.maxlen = size; + stringInfo.cursor = 0; + + LOCAL_FCINFO(callInfo, 3); + Zero(*callInfo); + callInfo->flinfo = (FmgrInfo*)&ReceiveFuncInfo; + callInfo->nargs = 3; + callInfo->fncollation = DEFAULT_COLLATION_OID; + callInfo->isnull = false; + callInfo->args[0] = { (Datum)&stringInfo, false }; + callInfo->args[1] = { ObjectIdGetDatum(NKikimr::NMiniKQL::MakeTypeIOParam(*this)), false }; + callInfo->args[2] = { Int32GetDatum(-1), false }; + + auto result = ReceiveFuncInfo.fn_addr(callInfo); + Y_ENSURE(!callInfo->isnull); + return result; + } + + static void InitFunc(ui32 funcId, FmgrInfo* info, ui32 argCountMin, ui32 argCountMax) { + Zero(*info); + Y_ENSURE(funcId); + fmgr_info(funcId, info); + Y_ENSURE(info->fn_addr); + Y_ENSURE(info->fn_nargs >= argCountMin && info->fn_nargs <= argCountMax); + } + +private: + FmgrInfo CompareProcInfo; + FmgrInfo HashProcInfo; + FmgrInfo ReceiveFuncInfo; + FmgrInfo SendFuncInfo; +}; + +class TPgTypeDescriptors { +public: + static const TPgTypeDescriptors& Instance() { + return *Singleton<TPgTypeDescriptors>(); + } + + TPgTypeDescriptors() { + auto initType = [this] (ui32 pgTypeId, const NYql::NPg::TTypeDesc& type) { + this->InitType(pgTypeId, type); + }; + NYql::NPg::EnumTypes(initType); + } + + const TPgTypeDescriptor* Find(ui32 pgTypeId) const { + return PgTypeDescriptors.FindPtr(pgTypeId); + } + + const TPgTypeDescriptor* Find(const TStringBuf name) const { + auto* id = ByName.FindPtr(name); + if (id) { + return Find(*id); + } + return {}; + } + +private: + void InitType(ui32 pgTypeId, const NYql::NPg::TTypeDesc& type) { + if (type.TypeId == type.ArrayTypeId) { // TODO: support arrays + return; + } + PgTypeDescriptors[pgTypeId] = TPgTypeDescriptor(type); + ByName[type.Name] = pgTypeId; + } + +private: + THashMap<ui32, TPgTypeDescriptor> PgTypeDescriptors; + THashMap<TString, ui32> ByName; +}; + +ui32 PgTypeIdFromTypeDesc(void* typeDesc) { + if (!typeDesc) { + return 0; + } + return static_cast<TPgTypeDescriptor*>(typeDesc)->TypeId; +} + +void* TypeDescFromPgTypeId(ui32 pgTypeId) { + if (!pgTypeId) { + return {}; + } + return (void*)TPgTypeDescriptors::Instance().Find(pgTypeId); +} + +const char* PgTypeNameFromTypeDesc(void* typeDesc) { + if (!typeDesc) { + return ""; + } + return static_cast<TPgTypeDescriptor*>(typeDesc)->Name.data(); +} + +void* TypeDescFromPgTypeName(const TStringBuf name) { + return (void*)TPgTypeDescriptors::Instance().Find(name); +} + +bool TypeDescIsComparable(void* typeDesc) { + if (!typeDesc) { + return false; + } + return static_cast<TPgTypeDescriptor*>(typeDesc)->CompareProcId != 0; +} + +ui32 TypeDescGetTypeLen(void* typeDesc) { + if (!typeDesc) { + return 0; + } + i32 res = static_cast<TPgTypeDescriptor*>(typeDesc)->TypeLen; + return res < 0 ? 0 : (ui32)res; +} + +int PgNativeBinaryCompare(const char* dataL, size_t sizeL, const char* dataR, size_t sizeR, void* typeDesc) { + return static_cast<TPgTypeDescriptor*>(typeDesc)->Compare(dataL, sizeL, dataR, sizeR); +} + +ui64 PgNativeBinaryHash(const char* data, size_t size, void* typeDesc) { + return static_cast<TPgTypeDescriptor*>(typeDesc)->Hash(data, size); +} + +} // namespace NKikimr::NPg + diff --git a/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp b/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp index 172483b1189..b10d6b4c1fd 100644 --- a/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp +++ b/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp @@ -54,7 +54,7 @@ bool IsRetriable(::NYdb::EStatus status) { } } -bool RangeFinished(const TString& lastReadKey, const TString& endKey, const TVector<NKikimr::NScheme::TTypeId>& keyColumnTypes) { +bool RangeFinished(const TString& lastReadKey, const TString& endKey, const TVector<NKikimr::NScheme::TTypeInfo>& keyColumnTypes) { if (lastReadKey.empty()) return true; @@ -78,7 +78,7 @@ public: const TString& path, ::NYdb::TDriver driver, const NActors::TActorId& computeActorId, - const TVector<TString>& columns, const TVector<NKikimr::NScheme::TTypeId>& keyColumnTypes, + const TVector<TString>& columns, const TVector<NKikimr::NScheme::TTypeInfo>& keyColumnTypes, ui64 maxRowsInRequest, ui64 maxBytesInRequest, const TString& keyFrom, const TString& keyTo ) : InputIndex(inputIndex) @@ -208,7 +208,7 @@ private: const TString Path; const TVector<TString> Columns; - const TVector<NKikimr::NScheme::TTypeId> KeyColumnTypes; + const TVector<NKikimr::NScheme::TTypeInfo> KeyColumnTypes; const ui64 MaxRows; const ui64 MaxBytes; const TString EndKey; @@ -253,10 +253,11 @@ std::pair<NYql::NDq::IDqComputeActorAsyncInput*, IActor*> CreateYdbReadActor( for (auto i = 0; i < params.GetColumns().size(); ++i) columns.emplace_back(params.GetColumns().Get(i)); - TVector<NKikimr::NScheme::TTypeId> keyColumnTypes; + TVector<NKikimr::NScheme::TTypeInfo> keyColumnTypes; keyColumnTypes.reserve(params.GetKeyColumnTypes().size()); for (auto i = 0; i < params.GetKeyColumnTypes().size(); ++i) - keyColumnTypes.emplace_back(params.GetKeyColumnTypes().Get(i)); + // TODO support pg types + keyColumnTypes.emplace_back(NKikimr::NScheme::TTypeInfo(params.GetKeyColumnTypes().Get(i), nullptr)); ui64 maxRowsInRequest = 0ULL; ui64 maxBytesInRequest = 0ULL; diff --git a/ydb/library/yql/providers/ydb/comp_nodes/yql_kik_scan.cpp b/ydb/library/yql/providers/ydb/comp_nodes/yql_kik_scan.cpp index f8fd6184b31..1e69f3346fd 100644 --- a/ydb/library/yql/providers/ydb/comp_nodes/yql_kik_scan.cpp +++ b/ydb/library/yql/providers/ydb/comp_nodes/yql_kik_scan.cpp @@ -32,7 +32,7 @@ bool IsRetriable(NYdb::EStatus status) { } } -bool RangeFinished(const TString& lastReadKey, const TString& endKey, const TVector<NKikimr::NScheme::TTypeId>& keyColumnTypes) { +bool RangeFinished(const TString& lastReadKey, const TString& endKey, const TVector<NKikimr::NScheme::TTypeInfo>& keyColumnTypes) { if (lastReadKey.empty()) return true; @@ -72,7 +72,7 @@ using TBaseComputation = TMutableComputationNode<TKikScan<Async>>; using TPtr = std::shared_ptr<TAsyncState>; using TWeakPtr = std::weak_ptr<TAsyncState>; - static TPtr Make(const NYdb::TDriver& driver, const TString& database, const TString& endpoint, const TString& token, const TString& path, bool secure, const TVector<TString>& columns, const TVector<NKikimr::NScheme::TTypeId>& keyColumnTypes, + static TPtr Make(const NYdb::TDriver& driver, const TString& database, const TString& endpoint, const TString& token, const TString& path, bool secure, const TVector<TString>& columns, const TVector<NKikimr::NScheme::TTypeInfo>& keyColumnTypes, ui64 maxRowsInRequest, ui64 maxBytesInRequest, const TString& keyFrom, const TString& keyTo, const NYdb::NClickhouseInternal::TScanSettings& settings) { const auto ptr = std::make_shared<TAsyncState>(driver, database, endpoint, token, path, secure, columns, keyColumnTypes, maxRowsInRequest, maxBytesInRequest, keyFrom, keyTo, settings); ptr->SendRequest(); @@ -100,7 +100,7 @@ using TBaseComputation = TMutableComputationNode<TKikScan<Async>>; return NUdf::EFetchStatus::Yield; } - TAsyncState(const NYdb::TDriver& driver, const TString& database, const TString& endpoint, const TString& token, const TString& path, bool secure, const TVector<TString>& columns, const TVector<NKikimr::NScheme::TTypeId>& keyColumnTypes, + TAsyncState(const NYdb::TDriver& driver, const TString& database, const TString& endpoint, const TString& token, const TString& path, bool secure, const TVector<TString>& columns, const TVector<NKikimr::NScheme::TTypeInfo>& keyColumnTypes, ui64 maxRowsInRequest, ui64 maxBytesInRequest, const TString& keyFrom, const TString& keyTo, const NYdb::NClickhouseInternal::TScanSettings& settings) : ActorSystem(NActors::TActivationContext::ActorSystem()) , CurrentActorId(NActors::TActivationContext::AsActorContext().SelfID) @@ -186,7 +186,7 @@ using TBaseComputation = TMutableComputationNode<TKikScan<Async>>; const TString Path; const TVector<TString> Columns; - const TVector<NKikimr::NScheme::TTypeId> KeyColumnTypes; + const TVector<NKikimr::NScheme::TTypeInfo> KeyColumnTypes; const ui64 MaxRows; const ui64 MaxBytes; const TString EndKey; @@ -208,7 +208,7 @@ using TBaseComputation = TMutableComputationNode<TKikScan<Async>>; TIssues Issues; }; public: - TAsyncStream(TMemoryUsageInfo* memInfo, const NYdb::TDriver& driver, const TString& database, const TString& endpoint, const TString& token, const TString& path, bool secure, const TVector<TString>& columns, const TVector<NKikimr::NScheme::TTypeId>& keyColumnTypes, + TAsyncStream(TMemoryUsageInfo* memInfo, const NYdb::TDriver& driver, const TString& database, const TString& endpoint, const TString& token, const TString& path, bool secure, const TVector<TString>& columns, const TVector<NKikimr::NScheme::TTypeInfo>& keyColumnTypes, ui64 maxRowsInRequest, ui64 maxBytesInRequest, const TString& keyFrom, const TString& keyTo, const NYdb::NClickhouseInternal::TScanSettings& settings) : TComputationValue<TAsyncStream>(memInfo), State(TAsyncState::Make(driver, database, endpoint, token, path, secure, columns, keyColumnTypes, maxRowsInRequest, maxBytesInRequest, keyFrom, keyTo, settings)) {} @@ -230,7 +230,7 @@ public: const std::string_view& snapshot, bool secure, TVector<TString>&& columns, - TVector<NKikimr::NScheme::TTypeId>&& keyColumnTypes, + TVector<NKikimr::NScheme::TTypeInfo>&& keyColumnTypes, const std::string_view& keyFrom, const std::string_view& keyTo, IComputationNode* rows @@ -287,7 +287,7 @@ private: const TString Snapshot; const bool Secure; const TVector<TString> Columns; - const TVector<NKikimr::NScheme::TTypeId> KeyColumnTypes; + const TVector<NKikimr::NScheme::TTypeInfo> KeyColumnTypes; const TString KeyFrom; const TString KeyTo; IComputationNode* const Rows; @@ -312,7 +312,7 @@ IComputationNode* WrapKikScan(TCallable& callable, const TComputationNodeFactory } const auto keysNode = AS_VALUE(TTupleLiteral, callable.GetInput(6)); - TVector<NKikimr::NScheme::TTypeId> keyColumnTypes; + TVector<NKikimr::NScheme::TTypeInfo> keyColumnTypes; keyColumnTypes.reserve(keysNode->GetValuesCount()); for (ui32 i = 0; i < keysNode->GetValuesCount(); ++i) { keyColumnTypes.emplace_back(AS_VALUE(TDataLiteral, keysNode->GetValue(i))->AsValue().Get<ui16>()); diff --git a/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp b/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp index 9758f72830f..999902d3269 100644 --- a/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp +++ b/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp @@ -258,3 +258,53 @@ NKikimr::NMiniKQL::TComputationNodeFactory GetPgFactory() { } } // NYql + +namespace NKikimr::NPg { + +ui32 PgTypeIdFromTypeDesc(void* typeDesc) { + Y_UNUSED(typeDesc); + return 0; +} + +void* TypeDescFromPgTypeId(ui32 pgTypeId) { + Y_UNUSED(pgTypeId); + return {}; +} + +const char* PgTypeNameFromTypeDesc(void* typeDesc) { + Y_UNUSED(typeDesc); + return ""; +} + +void* TypeDescFromPgTypeName(const TStringBuf name) { + Y_UNUSED(name); + return {}; +} + +bool TypeDescIsComparable(void* typeDesc) { + Y_UNUSED(typeDesc); + throw yexception() << "PG types are not supported"; +} + +ui32 TypeDescGetTypeLen(void* typeDesc) { + Y_UNUSED(typeDesc); + throw yexception() << "PG types are not supported"; +} + +int PgNativeBinaryCompare(const char* dataL, size_t sizeL, const char* dataR, size_t sizeR, void* typeDesc) { + Y_UNUSED(dataL); + Y_UNUSED(sizeL); + Y_UNUSED(dataR); + Y_UNUSED(sizeR); + Y_UNUSED(typeDesc); + throw yexception() << "PG types are not supported"; +} + +ui64 PgNativeBinaryHash(const char* data, size_t size, void* typeDesc) { + Y_UNUSED(data); + Y_UNUSED(size); + Y_UNUSED(typeDesc); + throw yexception() << "PG types are not supported"; +} + +} // namespace NKikimr::NPg diff --git a/ydb/library/yql/utils/actors/ut/CMakeLists.darwin.txt b/ydb/library/yql/utils/actors/ut/CMakeLists.darwin.txt index 64444e0a9fb..0f8ad586038 100644 --- a/ydb/library/yql/utils/actors/ut/CMakeLists.darwin.txt +++ b/ydb/library/yql/utils/actors/ut/CMakeLists.darwin.txt @@ -17,7 +17,7 @@ target_link_libraries(ydb-library-yql-utils-actors-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main yql-utils-actors - core-testlib-basics + testlib-basics-default ) target_link_options(ydb-library-yql-utils-actors-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/library/yql/utils/actors/ut/CMakeLists.linux.txt b/ydb/library/yql/utils/actors/ut/CMakeLists.linux.txt index efbdcff7e70..699d32cac69 100644 --- a/ydb/library/yql/utils/actors/ut/CMakeLists.linux.txt +++ b/ydb/library/yql/utils/actors/ut/CMakeLists.linux.txt @@ -19,7 +19,7 @@ target_link_libraries(ydb-library-yql-utils-actors-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main yql-utils-actors - core-testlib-basics + testlib-basics-default ) target_link_options(ydb-library-yql-utils-actors-ut PRIVATE -ldl diff --git a/ydb/library/yql/utils/pg_types.h b/ydb/library/yql/utils/pg_types.h new file mode 100644 index 00000000000..a043aca4ec9 --- /dev/null +++ b/ydb/library/yql/utils/pg_types.h @@ -0,0 +1,21 @@ +#pragma once + +#include <util/generic/strbuf.h> + +namespace NKikimr::NPg { + +ui32 PgTypeIdFromTypeDesc(void* typeDesc); +void* TypeDescFromPgTypeId(ui32 pgTypeId); + +const char* PgTypeNameFromTypeDesc(void* typeDesc); +void* TypeDescFromPgTypeName(const TStringBuf name); + +bool TypeDescIsComparable(void* typeDesc); +ui32 TypeDescGetTypeLen(void* typeDesc); + +int PgNativeBinaryCompare(const char* dataL, size_t sizeL, + const char* dataR, size_t sizeR, void* typeDesc); + +ui64 PgNativeBinaryHash(const char* data, size_t size, void* typeDesc); + +} // namespace NKikimr::NPg diff --git a/ydb/public/lib/deprecated/kicli/ut/CMakeLists.darwin.txt b/ydb/public/lib/deprecated/kicli/ut/CMakeLists.darwin.txt index e3e23b74f80..303a38ac602 100644 --- a/ydb/public/lib/deprecated/kicli/ut/CMakeLists.darwin.txt +++ b/ydb/public/lib/deprecated/kicli/ut/CMakeLists.darwin.txt @@ -21,7 +21,7 @@ target_link_libraries(ydb-public-lib-deprecated-kicli-ut PUBLIC cpp-testing-unittest_main lib-deprecated-kicli ydb-core-client - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-public-lib-deprecated-kicli-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/public/lib/deprecated/kicli/ut/CMakeLists.linux.txt b/ydb/public/lib/deprecated/kicli/ut/CMakeLists.linux.txt index 359e81eee84..f86ab54c216 100644 --- a/ydb/public/lib/deprecated/kicli/ut/CMakeLists.linux.txt +++ b/ydb/public/lib/deprecated/kicli/ut/CMakeLists.linux.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-public-lib-deprecated-kicli-ut PUBLIC cpp-testing-unittest_main lib-deprecated-kicli ydb-core-client - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-public-lib-deprecated-kicli-ut PRIVATE -ldl diff --git a/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp b/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp index bc47af36236..2566c84ec2a 100644 --- a/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp +++ b/ydb/public/lib/experimental/ydb_clickhouse_internal.cpp @@ -126,7 +126,7 @@ TAsyncScanResult TScanClient::Scan(const TString& path, const TVector<TString>& return Impl_->Scan(path, columns, maxRows, maxBytes, fromKey, fromKeyInclusive, settings); } -bool RangeFinished(const TString& lastReadKey, const TString& endKey, const TVector<NKikimr::NScheme::TTypeId>& keyColumnTypes) { +bool RangeFinished(const TString& lastReadKey, const TString& endKey, const TVector<NKikimr::NScheme::TTypeInfo>& keyColumnTypes) { if (lastReadKey.empty()) // +inf return true; @@ -149,7 +149,7 @@ bool RangeFinished(const TString& lastReadKey, const TString& endKey, const TVec TScanIterator::TScanIterator(const TDriver& driver, const TString &database, const TString &endpoint, const TString &token, bool ssl, const TString& path, const TVector<TString>& columns, - const TVector<NKikimr::NScheme::TTypeId>& keyColumnTypes, + const TVector<NKikimr::NScheme::TTypeInfo>& keyColumnTypes, ui64 maxRowsInRequest, ui64 maxBytesInRequest, const TString& keyFrom, const TString& keyTo, const TScanSettings& settings) @@ -172,7 +172,7 @@ TScanIterator::TScanIterator(const TDriver& driver, const TString &database, con TScanIterator::TScanIterator(const TDriver& driver, const TString &database, const TString &token, const TString& path, const TVector<TString>& columns, - const TVector<NKikimr::NScheme::TTypeId>& keyColumnTypes, + const TVector<NKikimr::NScheme::TTypeInfo>& keyColumnTypes, ui64 maxRowsInRequest, ui64 maxBytesInRequest, const TString& keyFrom, const TString& keyTo, const TScanSettings& settings) diff --git a/ydb/public/lib/experimental/ydb_clickhouse_internal.h b/ydb/public/lib/experimental/ydb_clickhouse_internal.h index d2a846ca8f8..26ab152ddd3 100644 --- a/ydb/public/lib/experimental/ydb_clickhouse_internal.h +++ b/ydb/public/lib/experimental/ydb_clickhouse_internal.h @@ -4,7 +4,7 @@ #include <ydb/public/api/grpc/draft/ydb_clickhouse_internal_v1.pb.h> // TODO: Bad dependency??? -#include <ydb/core/scheme/scheme_type_id.h> +#include <ydb/core/scheme/scheme_type_order.h> #include <util/generic/deque.h> @@ -61,12 +61,12 @@ private: class TScanIterator { public: TScanIterator(const TDriver& driver, const TString &database, const TString &endpoint, const TString& token, bool ssl, const TString& path, const TVector<TString>& columns, - const TVector<NKikimr::NScheme::TTypeId>& keyColumnTypes, + const TVector<NKikimr::NScheme::TTypeInfo>& keyColumnTypes, ui64 maxRowsInRequest, ui64 maxBytesInRequest, const TString& keyFrom = TString(), const TString& keyTo = TString(), const TScanSettings& settings = TScanSettings()); TScanIterator(const TDriver& driver, const TString &database, const TString& token, const TString& path, const TVector<TString>& columns, - const TVector<NKikimr::NScheme::TTypeId>& keyColumnTypes, + const TVector<NKikimr::NScheme::TTypeInfo>& keyColumnTypes, ui64 maxRowsInRequest, ui64 maxBytesInRequest, const TString& keyFrom = TString(), const TString& keyTo = TString(), const TScanSettings& settings = TScanSettings()); @@ -79,7 +79,7 @@ private: private: const TString Path; const TVector<TString> Columns; - const TVector<NKikimr::NScheme::TTypeId> KeyColumnTypes; + const TVector<NKikimr::NScheme::TTypeInfo> KeyColumnTypes; const ui64 MaxRows; const ui64 MaxBytes; const TScanSettings Settings; diff --git a/ydb/public/lib/scheme_types/scheme_type_id.h b/ydb/public/lib/scheme_types/scheme_type_id.h index 7f0bee58568..d18e4475f01 100644 --- a/ydb/public/lib/scheme_types/scheme_type_id.h +++ b/ydb/public/lib/scheme_types/scheme_type_id.h @@ -53,6 +53,8 @@ static constexpr TTypeId DyNumber = NYql::NProto::DyNumber; static constexpr TTypeId Decimal = NYql::NProto::Decimal; +static constexpr TTypeId Pg = 0x3000; + static constexpr TTypeId YqlIds[] = { Int32, Uint32, diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/CMakeLists.darwin.txt b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/CMakeLists.darwin.txt index 3b0fb0fa706..94ca427b333 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/CMakeLists.darwin.txt +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-public-sdk-cpp-client-ydb_persqueue_core-ut PUBLIC cpp-testing-unittest_main cpp-client-ydb_persqueue_core cpp-testing-gmock_in_unittest - ydb-core-testlib + core-testlib-default public-lib-json_value public-lib-yson_value cpp-client-ydb_driver diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/CMakeLists.linux.txt b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/CMakeLists.linux.txt index 3921d064eba..9a5b4ef6205 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/CMakeLists.linux.txt +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-public-sdk-cpp-client-ydb_persqueue_core-ut PUBLIC cpp-testing-unittest_main cpp-client-ydb_persqueue_core cpp-testing-gmock_in_unittest - ydb-core-testlib + core-testlib-default public-lib-json_value public-lib-yson_value cpp-client-ydb_driver diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/CMakeLists.txt b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/CMakeLists.txt index 95b05a2af53..802507fc68d 100644 --- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/CMakeLists.txt +++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/CMakeLists.txt @@ -17,7 +17,7 @@ target_link_libraries(ydb_persqueue_core-ut-ut_utils PUBLIC cpp-grpc-server cpp-testing-unittest cpp-threading-chunk_queue - ydb-core-testlib + core-testlib-default library-persqueue-topic_parser_public cpp-client-ydb_driver cpp-client-ydb_persqueue_core diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp index a79995ba177..c3de07b9501 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp @@ -1036,6 +1036,16 @@ TTableBuilder& TTableBuilder::AddNullableColumn(const TString& name, const TDeci return *this; } +TTableBuilder& TTableBuilder::AddNullableColumn(const TString& name, const TPgType& type, const TString& family) { + auto columnType = TTypeBuilder() + .BeginOptional() + .Pg(type) + .EndOptional() + .Build(); + TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family); + return *this; +} + TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const EPrimitiveType& type, const TString& family) { auto columnType = TTypeBuilder() .Primitive(type) @@ -1054,6 +1064,15 @@ TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const TD return *this; } +TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const TPgType& type, const TString& family) { + auto columnType = TTypeBuilder() + .Pg(type) + .Build(); + + TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family); + return *this; +} + TTableBuilder& TTableBuilder::SetPrimaryKeyColumns(const TVector<TString>& primaryKeyColumns) { TableDescription_.SetPrimaryKeyColumns(primaryKeyColumns); return *this; diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h index 54ece32d4cf..848cf5ca11d 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table.h @@ -684,8 +684,10 @@ public: TTableBuilder& AddNullableColumn(const TString& name, const EPrimitiveType& type, const TString& family = TString()); TTableBuilder& AddNullableColumn(const TString& name, const TDecimalType& type, const TString& family = TString()); + TTableBuilder& AddNullableColumn(const TString& name, const TPgType& type, const TString& family = TString()); TTableBuilder& AddNonNullableColumn(const TString& name, const EPrimitiveType& type, const TString& family = TString()); TTableBuilder& AddNonNullableColumn(const TString& name, const TDecimalType& type, const TString& family = TString()); + TTableBuilder& AddNonNullableColumn(const TString& name, const TPgType& type, const TString& family = TString()); TTableBuilder& SetPrimaryKeyColumns(const TVector<TString>& primaryKeyColumns); TTableBuilder& SetPrimaryKeyColumn(const TString& primaryKeyColumn); diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/CMakeLists.darwin.txt b/ydb/public/sdk/cpp/client/ydb_topic/ut/CMakeLists.darwin.txt index a950fc621c8..8699300a9b8 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ut/CMakeLists.darwin.txt +++ b/ydb/public/sdk/cpp/client/ydb_topic/ut/CMakeLists.darwin.txt @@ -21,7 +21,7 @@ target_link_libraries(ydb-public-sdk-cpp-client-ydb_topic-ut PUBLIC cpp-testing-unittest_main cpp-client-ydb_topic cpp-testing-gmock_in_unittest - ydb-core-testlib + core-testlib-default public-lib-json_value public-lib-yson_value cpp-client-ydb_driver diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/CMakeLists.linux.txt b/ydb/public/sdk/cpp/client/ydb_topic/ut/CMakeLists.linux.txt index 9a818366aae..a360ff19a59 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ut/CMakeLists.linux.txt +++ b/ydb/public/sdk/cpp/client/ydb_topic/ut/CMakeLists.linux.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-public-sdk-cpp-client-ydb_topic-ut PUBLIC cpp-testing-unittest_main cpp-client-ydb_topic cpp-testing-gmock_in_unittest - ydb-core-testlib + core-testlib-default public-lib-json_value public-lib-yson_value cpp-client-ydb_driver diff --git a/ydb/services/cms/ut/CMakeLists.darwin.txt b/ydb/services/cms/ut/CMakeLists.darwin.txt index 399c993510e..68b26c408cb 100644 --- a/ydb/services/cms/ut/CMakeLists.darwin.txt +++ b/ydb/services/cms/ut/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-services-cms-ut PUBLIC cpp-grpc-client cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-services-cms-ut PRIVATE -Wl,-no_deduplicate diff --git a/ydb/services/cms/ut/CMakeLists.linux.txt b/ydb/services/cms/ut/CMakeLists.linux.txt index cece96b6eaa..def6fd53d67 100644 --- a/ydb/services/cms/ut/CMakeLists.linux.txt +++ b/ydb/services/cms/ut/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-services-cms-ut PUBLIC cpp-grpc-client cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default ) target_link_options(ydb-services-cms-ut PRIVATE -ldl diff --git a/ydb/services/datastreams/ut/CMakeLists.darwin.txt b/ydb/services/datastreams/ut/CMakeLists.darwin.txt index cf5139a2994..f19d57819fa 100644 --- a/ydb/services/datastreams/ut/CMakeLists.darwin.txt +++ b/ydb/services/datastreams/ut/CMakeLists.darwin.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-services-datastreams-ut PUBLIC library-cpp-getopt cpp-grpc-client library-cpp-svnversion - ydb-core-testlib + core-testlib-default cpp-client-ydb_topic ) target_link_options(ydb-services-datastreams-ut PRIVATE diff --git a/ydb/services/datastreams/ut/CMakeLists.linux.txt b/ydb/services/datastreams/ut/CMakeLists.linux.txt index a946cf9c7b8..6ff6cc1187a 100644 --- a/ydb/services/datastreams/ut/CMakeLists.linux.txt +++ b/ydb/services/datastreams/ut/CMakeLists.linux.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-services-datastreams-ut PUBLIC library-cpp-getopt cpp-grpc-client library-cpp-svnversion - ydb-core-testlib + core-testlib-default cpp-client-ydb_topic ) target_link_options(ydb-services-datastreams-ut PRIVATE diff --git a/ydb/services/fq/ut_integration/CMakeLists.darwin.txt b/ydb/services/fq/ut_integration/CMakeLists.darwin.txt index b081c8cd61a..42b2074788d 100644 --- a/ydb/services/fq/ut_integration/CMakeLists.darwin.txt +++ b/ydb/services/fq/ut_integration/CMakeLists.darwin.txt @@ -24,12 +24,11 @@ target_link_libraries(ydb-services-fq-ut_integration PUBLIC cpp-grpc-client cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default yq-libs-control_plane_storage yq-libs-db_schema yq-libs-private_client providers-common-db_id_async_resolver - yql-sql-pg_dummy clickhouse_client_udf library-yql-utils public-lib-fq diff --git a/ydb/services/fq/ut_integration/CMakeLists.linux.txt b/ydb/services/fq/ut_integration/CMakeLists.linux.txt index a08124095b2..8d014843d46 100644 --- a/ydb/services/fq/ut_integration/CMakeLists.linux.txt +++ b/ydb/services/fq/ut_integration/CMakeLists.linux.txt @@ -26,12 +26,11 @@ target_link_libraries(ydb-services-fq-ut_integration PUBLIC cpp-grpc-client cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default yq-libs-control_plane_storage yq-libs-db_schema yq-libs-private_client providers-common-db_id_async_resolver - yql-sql-pg_dummy clickhouse_client_udf library-yql-utils public-lib-fq diff --git a/ydb/services/persqueue_cluster_discovery/ut/CMakeLists.darwin.txt b/ydb/services/persqueue_cluster_discovery/ut/CMakeLists.darwin.txt index 970df4a62d3..a35056c8344 100644 --- a/ydb/services/persqueue_cluster_discovery/ut/CMakeLists.darwin.txt +++ b/ydb/services/persqueue_cluster_discovery/ut/CMakeLists.darwin.txt @@ -21,7 +21,7 @@ target_link_libraries(ydb-services-persqueue_cluster_discovery-ut PUBLIC cpp-testing-unittest_main ydb-services-persqueue_cluster_discovery cpp-actors-http - ydb-core-testlib + core-testlib-default api-grpc ) target_link_options(ydb-services-persqueue_cluster_discovery-ut PRIVATE diff --git a/ydb/services/persqueue_cluster_discovery/ut/CMakeLists.linux.txt b/ydb/services/persqueue_cluster_discovery/ut/CMakeLists.linux.txt index 0625cbb15e5..23eaf963394 100644 --- a/ydb/services/persqueue_cluster_discovery/ut/CMakeLists.linux.txt +++ b/ydb/services/persqueue_cluster_discovery/ut/CMakeLists.linux.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-services-persqueue_cluster_discovery-ut PUBLIC cpp-testing-unittest_main ydb-services-persqueue_cluster_discovery cpp-actors-http - ydb-core-testlib + core-testlib-default api-grpc ) target_link_options(ydb-services-persqueue_cluster_discovery-ut PRIVATE diff --git a/ydb/services/persqueue_v1/ut/CMakeLists.darwin.txt b/ydb/services/persqueue_v1/ut/CMakeLists.darwin.txt index a3f22ab07a3..b1b003b5c0a 100644 --- a/ydb/services/persqueue_v1/ut/CMakeLists.darwin.txt +++ b/ydb/services/persqueue_v1/ut/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-services-persqueue_v1-ut PUBLIC library-cpp-getopt library-cpp-svnversion cpp-digest-md5 - ydb-core-testlib + core-testlib-default ydb-library-aclib library-persqueue-topic_parser api-grpc diff --git a/ydb/services/persqueue_v1/ut/CMakeLists.linux.txt b/ydb/services/persqueue_v1/ut/CMakeLists.linux.txt index cf4cd9f3a78..b7f1bcbfb8d 100644 --- a/ydb/services/persqueue_v1/ut/CMakeLists.linux.txt +++ b/ydb/services/persqueue_v1/ut/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-services-persqueue_v1-ut PUBLIC library-cpp-getopt library-cpp-svnversion cpp-digest-md5 - ydb-core-testlib + core-testlib-default ydb-library-aclib library-persqueue-topic_parser api-grpc diff --git a/ydb/services/persqueue_v1/ut/new_schemecache_ut/CMakeLists.darwin.txt b/ydb/services/persqueue_v1/ut/new_schemecache_ut/CMakeLists.darwin.txt index 9b03806b7c8..63e933b8541 100644 --- a/ydb/services/persqueue_v1/ut/new_schemecache_ut/CMakeLists.darwin.txt +++ b/ydb/services/persqueue_v1/ut/new_schemecache_ut/CMakeLists.darwin.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-services-persqueue_v1-ut-new_schemecache_ut PUBLIC ydb-services-persqueue_v1 library-cpp-getopt library-cpp-svnversion - ydb-core-testlib + core-testlib-default api-grpc cpp-client-resources ydb_persqueue_core-ut-ut_utils diff --git a/ydb/services/persqueue_v1/ut/new_schemecache_ut/CMakeLists.linux.txt b/ydb/services/persqueue_v1/ut/new_schemecache_ut/CMakeLists.linux.txt index 719d5b4e45c..fca6c752f28 100644 --- a/ydb/services/persqueue_v1/ut/new_schemecache_ut/CMakeLists.linux.txt +++ b/ydb/services/persqueue_v1/ut/new_schemecache_ut/CMakeLists.linux.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-services-persqueue_v1-ut-new_schemecache_ut PUBLIC ydb-services-persqueue_v1 library-cpp-getopt library-cpp-svnversion - ydb-core-testlib + core-testlib-default api-grpc cpp-client-resources ydb_persqueue_core-ut-ut_utils diff --git a/ydb/services/rate_limiter/ut/CMakeLists.darwin.txt b/ydb/services/rate_limiter/ut/CMakeLists.darwin.txt index 92bba5f85a5..d5fcf8b5cfc 100644 --- a/ydb/services/rate_limiter/ut/CMakeLists.darwin.txt +++ b/ydb/services/rate_limiter/ut/CMakeLists.darwin.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-services-rate_limiter-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main ydb-services-rate_limiter - ydb-core-testlib + core-testlib-default cpp-client-ydb_coordination cpp-client-ydb_rate_limiter ) diff --git a/ydb/services/rate_limiter/ut/CMakeLists.linux.txt b/ydb/services/rate_limiter/ut/CMakeLists.linux.txt index 29ed3804473..321cdf521bb 100644 --- a/ydb/services/rate_limiter/ut/CMakeLists.linux.txt +++ b/ydb/services/rate_limiter/ut/CMakeLists.linux.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-services-rate_limiter-ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main ydb-services-rate_limiter - ydb-core-testlib + core-testlib-default cpp-client-ydb_coordination cpp-client-ydb_rate_limiter ) diff --git a/ydb/services/ydb/index_ut/CMakeLists.darwin.txt b/ydb/services/ydb/index_ut/CMakeLists.darwin.txt index e8b77a793f6..ccb63930a85 100644 --- a/ydb/services/ydb/index_ut/CMakeLists.darwin.txt +++ b/ydb/services/ydb/index_ut/CMakeLists.darwin.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-services-ydb-index_ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main ydb-services-ydb - ydb-core-testlib + core-testlib-default public-lib-idx_test public-lib-yson_value cpp-client-ydb_scheme diff --git a/ydb/services/ydb/index_ut/CMakeLists.linux.txt b/ydb/services/ydb/index_ut/CMakeLists.linux.txt index ba32415c3f9..ab63c756d93 100644 --- a/ydb/services/ydb/index_ut/CMakeLists.linux.txt +++ b/ydb/services/ydb/index_ut/CMakeLists.linux.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-services-ydb-index_ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main ydb-services-ydb - ydb-core-testlib + core-testlib-default public-lib-idx_test public-lib-yson_value cpp-client-ydb_scheme diff --git a/ydb/services/ydb/sdk_credprovider_ut/CMakeLists.darwin.txt b/ydb/services/ydb/sdk_credprovider_ut/CMakeLists.darwin.txt index 47beb0dd7b0..0c7751320e4 100644 --- a/ydb/services/ydb/sdk_credprovider_ut/CMakeLists.darwin.txt +++ b/ydb/services/ydb/sdk_credprovider_ut/CMakeLists.darwin.txt @@ -20,7 +20,7 @@ target_link_libraries(ydb-services-ydb-sdk_credprovider_ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main ydb-services-ydb - ydb-core-testlib + core-testlib-default public-lib-yson_value cpp-client-ydb_scheme cpp-client-ydb_table diff --git a/ydb/services/ydb/sdk_credprovider_ut/CMakeLists.linux.txt b/ydb/services/ydb/sdk_credprovider_ut/CMakeLists.linux.txt index 6574233dbd9..5a866d9c29c 100644 --- a/ydb/services/ydb/sdk_credprovider_ut/CMakeLists.linux.txt +++ b/ydb/services/ydb/sdk_credprovider_ut/CMakeLists.linux.txt @@ -22,7 +22,7 @@ target_link_libraries(ydb-services-ydb-sdk_credprovider_ut PUBLIC library-cpp-cpuid_check cpp-testing-unittest_main ydb-services-ydb - ydb-core-testlib + core-testlib-default public-lib-yson_value cpp-client-ydb_scheme cpp-client-ydb_table diff --git a/ydb/services/ydb/ut/CMakeLists.darwin.txt b/ydb/services/ydb/ut/CMakeLists.darwin.txt index c79c299acec..1e8060eaff9 100644 --- a/ydb/services/ydb/ut/CMakeLists.darwin.txt +++ b/ydb/services/ydb/ut/CMakeLists.darwin.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-services-ydb-ut PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default yql-minikql-dom yql-minikql-jsonpath public-lib-experimental diff --git a/ydb/services/ydb/ut/CMakeLists.linux.txt b/ydb/services/ydb/ut/CMakeLists.linux.txt index 19738502783..b4c55437f16 100644 --- a/ydb/services/ydb/ut/CMakeLists.linux.txt +++ b/ydb/services/ydb/ut/CMakeLists.linux.txt @@ -28,7 +28,7 @@ target_link_libraries(ydb-services-ydb-ut PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - ydb-core-testlib + core-testlib-default yql-minikql-dom yql-minikql-jsonpath public-lib-experimental diff --git a/ydb/services/ydb/ydb_ut.cpp b/ydb/services/ydb/ydb_ut.cpp index 65d0827b658..acb0647aa61 100644 --- a/ydb/services/ydb/ydb_ut.cpp +++ b/ydb/services/ydb/ydb_ut.cpp @@ -3568,7 +3568,7 @@ void CheckTablePartitions(const TKikimrWithGrpcAndRootSchema &server, ui32 typeId = type.optional_type().item().type_id(); TString cellStr; - DbgPrintValue(cellStr, cell, typeId); + DbgPrintValue(cellStr, cell, NScheme::TTypeInfo(typeId)); TString valStr; switch (typeId) { diff --git a/ydb/services/yq/ut_integration/CMakeLists.darwin.txt b/ydb/services/yq/ut_integration/CMakeLists.darwin.txt index 2491d6e443f..abb861ed7cb 100644 --- a/ydb/services/yq/ut_integration/CMakeLists.darwin.txt +++ b/ydb/services/yq/ut_integration/CMakeLists.darwin.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-services-yq-ut_integration PUBLIC cpp-grpc-client cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default yq-libs-control_plane_storage yq-libs-db_schema yq-libs-private_client @@ -32,7 +32,6 @@ target_link_libraries(ydb-services-yq-ut_integration PUBLIC providers-common-db_id_async_resolver clickhouse_client_udf library-yql-utils - yql-sql-pg_dummy ) target_link_options(ydb-services-yq-ut_integration PRIVATE -Wl,-no_deduplicate diff --git a/ydb/services/yq/ut_integration/CMakeLists.linux.txt b/ydb/services/yq/ut_integration/CMakeLists.linux.txt index 7a27e028331..4c82e4b35ee 100644 --- a/ydb/services/yq/ut_integration/CMakeLists.linux.txt +++ b/ydb/services/yq/ut_integration/CMakeLists.linux.txt @@ -26,7 +26,7 @@ target_link_libraries(ydb-services-yq-ut_integration PUBLIC cpp-grpc-client cpp-regex-pcre library-cpp-svnversion - ydb-core-testlib + core-testlib-default yq-libs-control_plane_storage yq-libs-db_schema yq-libs-private_client @@ -34,7 +34,6 @@ target_link_libraries(ydb-services-yq-ut_integration PUBLIC providers-common-db_id_async_resolver clickhouse_client_udf library-yql-utils - yql-sql-pg_dummy ) target_link_options(ydb-services-yq-ut_integration PRIVATE -ldl diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema index 3595c4ceed5..247e9e1f645 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema @@ -435,6 +435,11 @@ "ColumnId": 11, "ColumnName": "NotNull", "ColumnType": "Bool" + }, + { + "ColumnId": 12, + "ColumnName": "ColTypeData", + "ColumnType": "String" } ], "ColumnsDropped": [], @@ -451,7 +456,8 @@ 8, 9, 10, - 11 + 11, + 12 ], "RoomID": 0, "Codec": 0, @@ -918,6 +924,11 @@ "ColumnId": 11, "ColumnName": "NotNull", "ColumnType": "Bool" + }, + { + "ColumnId": 12, + "ColumnName": "ColTypeData", + "ColumnType": "String" } ], "ColumnsDropped": [], @@ -934,7 +945,8 @@ 8, 9, 10, - 11 + 11, + 12 ], "RoomID": 0, "Codec": 0, @@ -3539,6 +3551,11 @@ "ColumnId": 12, "ColumnName": "NotNull", "ColumnType": "Bool" + }, + { + "ColumnId": 13, + "ColumnName": "ColTypeData", + "ColumnType": "String" } ], "ColumnsDropped": [], @@ -3556,7 +3573,8 @@ 9, 10, 11, - 12 + 12, + 13 ], "RoomID": 0, "Codec": 0, @@ -4026,6 +4044,11 @@ "ColumnId": 12, "ColumnName": "NotNull", "ColumnType": "Bool" + }, + { + "ColumnId": 13, + "ColumnName": "ColTypeData", + "ColumnType": "String" } ], "ColumnsDropped": [], @@ -4043,7 +4066,8 @@ 9, 10, 11, - 12 + 12, + 13 ], "RoomID": 0, "Codec": 0, |