diff options
author | ijon <ijon@ydb.tech> | 2023-11-23 18:28:53 +0300 |
---|---|---|
committer | ijon <ijon@ydb.tech> | 2023-11-23 19:16:12 +0300 |
commit | 36d5794c5f59dab0826acad3c7d3507b9e540968 (patch) | |
tree | 6973b2712c55358e506feb2d8465c1d71a6e9fd6 | |
parent | a37f1619d044459e0e99224a5f0c8ca510154816 (diff) | |
download | ydb-36d5794c5f59dab0826acad3c7d3507b9e540968.tar.gz |
schemeshard: add EnableTablePgTypes feature flag to disable use of pg types in table columns
KIKIMR-20226
20 files changed, 92 insertions, 13 deletions
diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.cpp b/ydb/core/kqp/ut/common/kqp_ut_common.cpp index 1d8bc3d176..cd4bf4be82 100644 --- a/ydb/core/kqp/ut/common/kqp_ut_common.cpp +++ b/ydb/core/kqp/ut/common/kqp_ut_common.cpp @@ -125,6 +125,7 @@ TKikimrRunner::TKikimrRunner(const TKikimrSettings& settings) { ServerSettings->SetEnableMoveIndex(true); ServerSettings->SetEnableUniqConstraint(true); ServerSettings->SetUseRealThreads(settings.UseRealThreads); + ServerSettings->SetEnableTablePgTypes(true); if (settings.Storage) { ServerSettings->SetCustomDiskParams(*settings.Storage); @@ -468,7 +469,7 @@ void TKikimrRunner::Initialize(const TKikimrSettings& settings) { this->Client->InitRootScheme(domain); return true; }); - + if (settings.WithSampleTables) { RunCall([this] { this->CreateSampleTables(); @@ -670,7 +671,7 @@ void CreateManyShardsTable(TKikimrRunner& kikimr, ui32 totalRows, ui32 shards, u Columns { Name: "Key", Type: "Uint32" } Columns { Name: "Data", Type: "Int32" } KeyColumnNames: ["Key"] - UniformPartitionsCount: + UniformPartitionsCount: )" + std::to_string(shards)); auto client = kikimr.GetTableClient(); diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index 66de8709de..dcf2218f2c 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -120,4 +120,5 @@ message TFeatureFlags { optional bool EnableChangefeedDebeziumJsonFormat = 105 [default = false]; optional bool EnableStatistics = 106 [default = false]; optional bool EnableUuidAsPrimaryKey = 107 [default = false]; + optional bool EnableTablePgTypes = 108 [default = false]; } diff --git a/ydb/core/testlib/basics/feature_flags.h b/ydb/core/testlib/basics/feature_flags.h index 00bf98e982..ccd1e6fd46 100644 --- a/ydb/core/testlib/basics/feature_flags.h +++ b/ydb/core/testlib/basics/feature_flags.h @@ -54,6 +54,7 @@ public: FEATURE_FLAG_SETTER(EnableUniqConstraint) FEATURE_FLAG_SETTER(EnableTopicMessageMeta) FEATURE_FLAG_SETTER(EnableUuidAsPrimaryKey) + FEATURE_FLAG_SETTER(EnableTablePgTypes) #undef FEATURE_FLAG_SETTER }; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp index 00a3dce32e..2aa57064a0 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_table.cpp @@ -120,7 +120,7 @@ TTableInfo::TAlterDataPtr ParseParams(const TPath& path, TTableInfo::TPtr table, const TSubDomainInfo& subDomain = *path.DomainInfo(); const TSchemeLimits& limits = subDomain.GetSchemeLimits(); - TTableInfo::TAlterDataPtr alterData = TTableInfo::CreateAlterData(table, copyAlter, *appData->TypeRegistry, limits, subDomain, errStr); + TTableInfo::TAlterDataPtr alterData = TTableInfo::CreateAlterData(table, copyAlter, *appData->TypeRegistry, limits, subDomain, context.SS->EnableTablePgTypes, errStr); if (!alterData) { status = NKikimrScheme::StatusInvalidParameter; return nullptr; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp index e4826d108a..d67b2bf91a 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp @@ -459,7 +459,7 @@ public: const NScheme::TTypeRegistry* typeRegistry = AppData()->TypeRegistry; const TSchemeLimits& limits = domainInfo->GetSchemeLimits(); - TTableInfo::TAlterDataPtr alterData = TTableInfo::CreateAlterData(nullptr, schema, *typeRegistry, limits, *domainInfo, errStr); + TTableInfo::TAlterDataPtr alterData = TTableInfo::CreateAlterData(nullptr, schema, *typeRegistry, limits, *domainInfo, context.SS->EnableTablePgTypes, errStr); if (!alterData.Get()) { result->SetError(NKikimrScheme::StatusSchemeError, errStr); return result; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp index 1528bfa4dd..bb549e76de 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp @@ -101,7 +101,6 @@ bool InitPartitioning(const NKikimrSchemeOp::TTableDescription& op, return true; } - bool DoInitPartitioning(TTableInfo::TPtr tableInfo, const NKikimrSchemeOp::TTableDescription& op, const NScheme::TTypeRegistry* typeRegistry, @@ -564,7 +563,7 @@ public: const NScheme::TTypeRegistry* typeRegistry = AppData()->TypeRegistry; const TSchemeLimits& limits = domainInfo->GetSchemeLimits(); - TTableInfo::TAlterDataPtr alterData = TTableInfo::CreateAlterData(nullptr, schema, *typeRegistry, limits, *domainInfo, errStr, LocalSequences); + TTableInfo::TAlterDataPtr alterData = TTableInfo::CreateAlterData(nullptr, schema, *typeRegistry, limits, *domainInfo, context.SS->EnableTablePgTypes, errStr, LocalSequences); if (!alterData.Get()) { result->SetError(NKikimrScheme::StatusSchemeError, errStr); return result; diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index c4972c3533..df5db089f8 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -4180,6 +4180,7 @@ void TSchemeShard::OnActivateExecutor(const TActorContext &ctx) { EnableAlterDatabaseCreateHiveFirst = appData->FeatureFlags.GetEnableAlterDatabaseCreateHiveFirst(); EnablePQConfigTransactionsAtSchemeShard = appData->FeatureFlags.GetEnablePQConfigTransactionsAtSchemeShard(); EnableStatistics = appData->FeatureFlags.GetEnableStatistics(); + EnableTablePgTypes = appData->FeatureFlags.GetEnableTablePgTypes(); ConfigureCompactionQueues(appData->CompactionConfig, ctx); ConfigureStatsBatching(appData->SchemeShardConfig, ctx); @@ -6627,6 +6628,7 @@ void TSchemeShard::ApplyConsoleConfigs(const NKikimrConfig::TFeatureFlags& featu EnableAlterDatabaseCreateHiveFirst = featureFlags.GetEnableAlterDatabaseCreateHiveFirst(); EnablePQConfigTransactionsAtSchemeShard = featureFlags.GetEnablePQConfigTransactionsAtSchemeShard(); EnableStatistics = featureFlags.GetEnableStatistics(); + EnableTablePgTypes = featureFlags.GetEnableTablePgTypes(); } void TSchemeShard::ConfigureStatsBatching(const NKikimrConfig::TSchemeShardConfig& config, const TActorContext& ctx) { diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index 8f5f729916..b12b29395f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -267,6 +267,7 @@ public: bool EnableAlterDatabaseCreateHiveFirst = false; bool EnablePQConfigTransactionsAtSchemeShard = false; bool EnableStatistics = false; + bool EnableTablePgTypes = false; TShardDeleter ShardDeleter; diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp index 56dfe9eee4..d3b4dcc472 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp @@ -51,6 +51,7 @@ TTableInfo::TAlterDataPtr TTableInfo::CreateAlterData( NKikimrSchemeOp::TTableDescription& op, const NScheme::TTypeRegistry& typeRegistry, const TSchemeLimits& limits, const TSubDomainInfo& subDomain, + bool pgTypesEnabled, TString& errStr, const THashSet<TString>& localSequences) { TAlterDataPtr alterData = new TTableInfo::TAlterTableInfo(); @@ -164,6 +165,10 @@ TTableInfo::TAlterDataPtr TTableInfo::CreateAlterData( errStr = Sprintf("Type '%s' specified for column '%s' is not supported by storage", col.GetType().data(), colName.data()); return nullptr; } + if (!pgTypesEnabled) { + errStr = Sprintf("Type '%s' specified for column '%s', but support for pg types is disabled (EnableTablePgTypes feature flag is off)", col.GetType().data(), colName.data()); + return nullptr; + } typeInfo = NScheme::TTypeInfo(NScheme::NTypeIds::Pg, typeDesc); typeMod = NPg::TypeModFromPgTypeName(typeName); } diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 0f746ccf06..87a55d4034 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -527,6 +527,7 @@ public: NKikimrSchemeOp::TTableDescription& descr, const NScheme::TTypeRegistry& typeRegistry, const TSchemeLimits& limits, const TSubDomainInfo& subDomain, + bool pgTypesEnabled, TString& errStr, const THashSet<TString>& localSequences = {}); static ui32 ShardsToCreate(const NKikimrSchemeOp::TTableDescription& descr) { diff --git a/ydb/core/tx/schemeshard/ut_base/CMakeLists.darwin-arm64.txt b/ydb/core/tx/schemeshard/ut_base/CMakeLists.darwin-arm64.txt index 0d8188bad6..7dab7ff10e 100644 --- a/ydb/core/tx/schemeshard/ut_base/CMakeLists.darwin-arm64.txt +++ b/ydb/core/tx/schemeshard/ut_base/CMakeLists.darwin-arm64.txt @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_base PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - core-testlib-default + core-testlib-pg ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy @@ -39,6 +39,7 @@ target_sources(ydb-core-tx-schemeshard-ut_base PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_base.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_info_types.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_allocate_pq.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_table_pg_types.cpp ) set_property( TARGET diff --git a/ydb/core/tx/schemeshard/ut_base/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/schemeshard/ut_base/CMakeLists.darwin-x86_64.txt index a6e1cbfa68..5f7f530d90 100644 --- a/ydb/core/tx/schemeshard/ut_base/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/tx/schemeshard/ut_base/CMakeLists.darwin-x86_64.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_base PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - core-testlib-default + core-testlib-pg ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy @@ -40,6 +40,7 @@ target_sources(ydb-core-tx-schemeshard-ut_base PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_base.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_info_types.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_allocate_pq.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_table_pg_types.cpp ) set_property( TARGET diff --git a/ydb/core/tx/schemeshard/ut_base/CMakeLists.linux-aarch64.txt b/ydb/core/tx/schemeshard/ut_base/CMakeLists.linux-aarch64.txt index b07c721714..8e895f64fc 100644 --- a/ydb/core/tx/schemeshard/ut_base/CMakeLists.linux-aarch64.txt +++ b/ydb/core/tx/schemeshard/ut_base/CMakeLists.linux-aarch64.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_base PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - core-testlib-default + core-testlib-pg ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy @@ -43,6 +43,7 @@ target_sources(ydb-core-tx-schemeshard-ut_base PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_base.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_info_types.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_allocate_pq.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_table_pg_types.cpp ) set_property( TARGET diff --git a/ydb/core/tx/schemeshard/ut_base/CMakeLists.linux-x86_64.txt b/ydb/core/tx/schemeshard/ut_base/CMakeLists.linux-x86_64.txt index c7720356f2..321ed8352e 100644 --- a/ydb/core/tx/schemeshard/ut_base/CMakeLists.linux-x86_64.txt +++ b/ydb/core/tx/schemeshard/ut_base/CMakeLists.linux-x86_64.txt @@ -25,7 +25,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_base PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - core-testlib-default + core-testlib-pg ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy @@ -44,6 +44,7 @@ target_sources(ydb-core-tx-schemeshard-ut_base PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_base.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_info_types.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_allocate_pq.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_table_pg_types.cpp ) set_property( TARGET diff --git a/ydb/core/tx/schemeshard/ut_base/CMakeLists.windows-x86_64.txt b/ydb/core/tx/schemeshard/ut_base/CMakeLists.windows-x86_64.txt index 69cb46db0d..88db1730cc 100644 --- a/ydb/core/tx/schemeshard/ut_base/CMakeLists.windows-x86_64.txt +++ b/ydb/core/tx/schemeshard/ut_base/CMakeLists.windows-x86_64.txt @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_base PUBLIC cpp-regex-pcre library-cpp-svnversion kqp-ut-common - core-testlib-default + core-testlib-pg ydb-core-tx tx-schemeshard-ut_helpers udf-service-exception_policy @@ -33,6 +33,7 @@ target_sources(ydb-core-tx-schemeshard-ut_base PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_base.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_info_types.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_allocate_pq.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_base/ut_table_pg_types.cpp ) set_property( TARGET diff --git a/ydb/core/tx/schemeshard/ut_base/ut_table_pg_types.cpp b/ydb/core/tx/schemeshard/ut_base/ut_table_pg_types.cpp new file mode 100644 index 0000000000..606ed5a621 --- /dev/null +++ b/ydb/core/tx/schemeshard/ut_base/ut_table_pg_types.cpp @@ -0,0 +1,60 @@ +#include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> + +using namespace NKikimr; +using namespace NSchemeShard; +using namespace NSchemeShardUT_Private; + +Y_UNIT_TEST_SUITE(TSchemeShardPgTypesInTables) { + + Y_UNIT_TEST_FLAG(CreateTableWithPgTypeColumn, EnableTablePgTypes) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableTablePgTypes(EnableTablePgTypes)); + ui64 txId = 100; + + auto expectedResult = []() { + if constexpr (EnableTablePgTypes) { + return TExpectedResult(NKikimrScheme::StatusAccepted); + } else { + return TExpectedResult(NKikimrScheme::StatusSchemeError, "support for pg types is disabled (EnableTablePgTypes feature flag is off)"); + } + }(); + + AsyncCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table1" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "pgint4" } + KeyColumnNames: ["key"] + )"); + TestModificationResults(runtime, txId, {expectedResult}); + env.TestWaitNotification(runtime, txId); + } + + Y_UNIT_TEST_FLAG(AlterTableAddPgTypeColumn, EnableTablePgTypes) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableTablePgTypes(EnableTablePgTypes)); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table1" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Uint8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + auto expectedResult = []() { + if constexpr (EnableTablePgTypes) { + return TExpectedResult(NKikimrScheme::StatusAccepted); + } else { + return TExpectedResult(NKikimrScheme::StatusInvalidParameter, "support for pg types is disabled (EnableTablePgTypes feature flag is off)"); + } + }(); + + AsyncAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "Table1" + Columns { Name: "added" Type: "pgint4" } + )"); + TestModificationResults(runtime, txId, {expectedResult}); + env.TestWaitNotification(runtime, txId); + } +} diff --git a/ydb/core/tx/schemeshard/ut_base/ya.make b/ydb/core/tx/schemeshard/ut_base/ya.make index ff632bb14e..d6716aa43a 100644 --- a/ydb/core/tx/schemeshard/ut_base/ya.make +++ b/ydb/core/tx/schemeshard/ut_base/ya.make @@ -15,7 +15,7 @@ PEERDIR( library/cpp/regex/pcre library/cpp/svnversion ydb/core/kqp/ut/common - ydb/core/testlib/default + ydb/core/testlib/pg ydb/core/tx ydb/core/tx/schemeshard/ut_helpers ydb/library/yql/public/udf/service/exception_policy @@ -27,6 +27,7 @@ SRCS( ut_base.cpp ut_info_types.cpp ut_allocate_pq.cpp + ut_table_pg_types.cpp ) END() diff --git a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp index 72358593e9..724a3ec21a 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp @@ -534,6 +534,7 @@ NSchemeShardUT_Private::TTestEnv::TTestEnv(TTestActorRuntime& runtime, const TTe app.SetEnableTopicSplitMerge(opts.EnableTopicSplitMerge_); app.SetEnableChangefeedDynamoDBStreamsFormat(opts.EnableChangefeedDynamoDBStreamsFormat_); app.SetEnableChangefeedDebeziumJsonFormat(opts.EnableChangefeedDebeziumJsonFormat_); + app.SetEnableTablePgTypes(opts.EnableTablePgTypes_); app.ColumnShardConfig.SetDisabledOnSchemeShard(false); diff --git a/ydb/core/tx/schemeshard/ut_helpers/test_env.h b/ydb/core/tx/schemeshard/ut_helpers/test_env.h index d726c73a78..2cce554c1b 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/test_env.h +++ b/ydb/core/tx/schemeshard/ut_helpers/test_env.h @@ -56,6 +56,7 @@ namespace NSchemeShardUT_Private { OPTION(std::optional<bool>, EnableTopicSplitMerge, std::nullopt); OPTION(std::optional<bool>, EnableChangefeedDynamoDBStreamsFormat, std::nullopt); OPTION(std::optional<bool>, EnableChangefeedDebeziumJsonFormat, std::nullopt); + OPTION(std::optional<bool>, EnableTablePgTypes, std::nullopt); #undef OPTION }; diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index 02b7089395..760b887b14 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -731,7 +731,7 @@ Y_UNIT_TEST_SUITE(TRestoreTests) { Y_UNIT_TEST(ExportImportPg) { TTestBasicRuntime runtime; - TTestEnv env(runtime); + TTestEnv env(runtime, TTestEnvOptions().EnableTablePgTypes(true)); ui64 txId = 100; TestCreateTable(runtime, ++txId, "/MyRoot", R"( |