diff options
author | nsofya <nsofya@yandex-team.com> | 2023-03-15 12:55:05 +0300 |
---|---|---|
committer | nsofya <nsofya@yandex-team.com> | 2023-03-15 12:55:05 +0300 |
commit | 720ab09aac0cd54c0b888ecdb0c53899f11daaee (patch) | |
tree | c903a6746ebfa90f6515da8681c5d7f3c79bdcf7 | |
parent | 7a343b35aea0c6cda407a7f0f11a2a494c8125b8 (diff) | |
download | ydb-720ab09aac0cd54c0b888ecdb0c53899f11daaee.tar.gz |
Show errors for not implemented alter requests
Что сделала:
1) Тест на обновление схемы колонок, который проверяет, что вернем ошибку
2) Перегруппировала текущий код, чтобы вся валидация входящего запроса была в одном классе
3) Максимально дополнила конвертер из TTableDescription в TAlterColumnTable
4) Сделала поля для Alter и Drop одного типа, аналогично даташарду.
Упал тест: https://a.yandex-team.ru/arcadia/ydb/core/tx/schemeshard/ut_olap.cpp?rev=b100a39763180844768c40940c1e2863fd140ee3#L568
Но насколько я видела из кода - это синтетический тест, по факту эти операции нельзя было произвести, верно? Ну и все равно, кажется, что переименование тут не критично, но поправьте, если я чего-то не поняла.
-rw-r--r-- | ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp | 32 | ||||
-rw-r--r-- | ydb/core/protos/flat_scheme_op.proto | 18 | ||||
-rw-r--r-- | ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp | 312 | ||||
-rw-r--r-- | ydb/core/tx/schemeshard/schemeshard_tables_storage.h | 3 | ||||
-rw-r--r-- | ydb/core/tx/schemeshard/ut_olap.cpp | 2 |
5 files changed, 226 insertions, 141 deletions
diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index 699f0d359c5..1f006b53864 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -3269,6 +3269,38 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } + Y_UNIT_TEST(AddColumnOlapTable) { + TKikimrSettings runnerSettings; + runnerSettings.WithSampleTables = false; + TKikimrRunner kikimr(runnerSettings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/ColumnTableTest"; + + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64 NOT NULL, + Value1 String, + Value2 Int64 NOT NULL, + PRIMARY KEY (Key) + ) + PARTITION BY HASH(Value1, Value2) + WITH ( + STORE = COLUMN, + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10 + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto query2 = TStringBuilder() << R"( + --!syntax_v1 + ALTER TABLE `)" << tableName << R"(`ADD COLUMN Value3 Uint64;)"; + result = session.ExecuteSchemeQuery(query2).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SCHEME_ERROR, result.GetIssues().ToString()); + + } + Y_UNIT_TEST(CreateDropColumnTableNegative) { TKikimrSettings runnerSettings; runnerSettings.WithSampleTables = false; diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 3d6a0be1d8d..eb3ed2e7cb6 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -380,18 +380,6 @@ message TOlapColumnDescription { optional bool NotNull = 7; } -message TAlterOlapColumn { - // Existing column name to alter - optional string Name = 1; - - // TODO: there's currently nothing to alter -} - -message TDropOlapColumn { - // Existing column name to drop - optional string Name = 1; -} - enum EColumnTableEngine { COLUMN_ENGINE_NONE = 0; COLUMN_ENGINE_REPLACING_TIMESERIES = 1; @@ -428,10 +416,8 @@ message TColumnTableSchema { } message TAlterColumnTableSchema { - repeated TOlapColumnDescription AddColumns = 1; - repeated TAlterOlapColumn AlterColumns = 2; - repeated TDropOlapColumn DropColumns = 3; - repeated string AddKeyColumnNames = 4; + repeated TOlapColumnDescription Columns = 1; + repeated TOlapColumnDescription DropColumns = 6; //optional TCompressionOptions DefaultCompression = 5; } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp index 2af196cc3ab..dd110ab865d 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp @@ -9,113 +9,186 @@ namespace { using namespace NKikimr; using namespace NSchemeShard; -NKikimrSchemeOp::TAlterColumnTable ConvertAlter(const NKikimrSchemeOp::TTableDescription& alterTable, - TString& /*errStr*/) { - NKikimrSchemeOp::TAlterColumnTable alter; - alter.SetName(alterTable.GetName()); - - // TODO: optional TAlterColumnTableSchema AlterSchema - - if (alterTable.HasTTLSettings()) { - auto& tableTtl = alterTable.GetTTLSettings(); - NKikimrSchemeOp::TColumnDataLifeCycle* alterTtl = alter.MutableAlterTtlSettings(); - if (tableTtl.HasEnabled()) { - auto& enabled = tableTtl.GetEnabled(); - auto* alterEnabled = alterTtl->MutableEnabled(); - if (enabled.HasColumnName()) { - alterEnabled->SetColumnName(enabled.GetColumnName()); + +class TTableInfoConstructor { + const NKikimrSchemeOp::TModifyScheme& ModifyRequest; + NKikimrSchemeOp::TAlterColumnTable AlterRequest; + +public: + TTableInfoConstructor(const NKikimrSchemeOp::TModifyScheme& modify) + : ModifyRequest(modify) {} + + const TString& GetTableName() const { + return AlterRequest.GetName(); + } + + bool CheckAndPrepare(TProposeResponse& errors) { + if (ModifyRequest.HasAlterColumnTable()) { + if (ModifyRequest.GetOperationType() != NKikimrSchemeOp::ESchemeOpAlterColumnTable) { + errors.SetError(NKikimrScheme::StatusSchemeError, "Invalid operation type"); + return false; } - if (enabled.HasExpireAfterSeconds()) { - alterEnabled->SetExpireAfterSeconds(enabled.GetExpireAfterSeconds()); + AlterRequest = ModifyRequest.GetAlterColumnTable(); + } else { + // from DDL (not known table type) + if (ModifyRequest.GetOperationType() != NKikimrSchemeOp::ESchemeOpAlterTable) { + errors.SetError(NKikimrScheme::StatusSchemeError, "Invalid operation type"); + return false; } - if (enabled.HasColumnUnit()) { - alterEnabled->SetColumnUnit(enabled.GetColumnUnit()); + if (!ParseFromDSRequest(ModifyRequest.GetAlterTable(), AlterRequest, errors)) { + return false; } - } else if (tableTtl.HasDisabled()) { - alterTtl->MutableDisabled(); } - if (tableTtl.HasUseTiering()) { - alterTtl->SetUseTiering(tableTtl.GetUseTiering()); + + if (!AlterRequest.HasName()) { + errors.SetError(NKikimrScheme::StatusInvalidParameter, "No table name in Alter"); + return false; } - } - return alter; -} + if (AlterRequest.HasAlterSchema() || AlterRequest.HasAlterSchemaPresetName()) { + errors.SetError(NKikimrScheme::StatusSchemeError, "Changing table schema is not supported"); + return false; + } -TColumnTableInfo::TPtr ParseParams( - const TPath& path, TTablesStorage::TTableExtractedGuard& tableInfo, const TOlapStoreInfo::TPtr& storeInfo, - const NKikimrSchemeOp::TAlterColumnTable& alter, TString& errStr) -{ - Y_UNUSED(path); + if (AlterRequest.HasRESERVED_AlterTtlSettingsPresetName()) { + errors.SetError(NKikimrScheme::StatusSchemeError, "TTL presets are not supported"); + return false; + } - if (alter.HasAlterSchema() || alter.HasAlterSchemaPresetName()) { - errStr = "Changing table schema is not supported"; - return nullptr; + return true; } - if (alter.HasRESERVED_AlterTtlSettingsPresetName()) { - errStr = "TTL presets are not supported"; - return nullptr; - } + TColumnTableInfo::TPtr BuildTableinfo(const TTablesStorage::TTableExtractedGuard& tableInfo, const TOlapStoreInfo::TPtr& storeInfo, TProposeResponse& errors) const { + TColumnTableInfo::TPtr alterData = new TColumnTableInfo(*tableInfo); + alterData->AlterBody.ConstructInPlace(AlterRequest); + ++alterData->AlterVersion; - TColumnTableInfo::TPtr alterData = new TColumnTableInfo(*tableInfo); - alterData->AlterBody.ConstructInPlace(alter); - ++alterData->AlterVersion; + ui64 currentTtlVersion = 0; + if (alterData->Description.HasTtlSettings()) { + currentTtlVersion = alterData->Description.GetTtlSettings().GetVersion(); + } - ui64 currentTtlVersion = 0; - if (alterData->Description.HasTtlSettings()) { - currentTtlVersion = alterData->Description.GetTtlSettings().GetVersion(); - } + const NKikimrSchemeOp::TColumnTableSchema* tableSchema = nullptr; + if (storeInfo) { + if (!storeInfo->SchemaPresets.count(tableInfo->Description.GetSchemaPresetId())) { + errors.SetError(NKikimrScheme::StatusSchemeError, "No preset for in-store column table"); + return nullptr; + } - const NKikimrSchemeOp::TColumnTableSchema* tableSchema = nullptr; - if (storeInfo) { - if (!storeInfo->SchemaPresets.count(tableInfo->Description.GetSchemaPresetId())) { - errStr = "No preset for in-store column table"; - return nullptr; - } + auto& preset = storeInfo->SchemaPresets.at(tableInfo->Description.GetSchemaPresetId()); + auto& presetProto = storeInfo->Description.GetSchemaPresets(preset.ProtoIndex); + if (!presetProto.HasSchema()) { + errors.SetError(NKikimrScheme::StatusSchemeError, "No schema in preset for in-store column table"); + return nullptr; + } - auto& preset = storeInfo->SchemaPresets.at(tableInfo->Description.GetSchemaPresetId()); - auto& presetProto = storeInfo->Description.GetSchemaPresets(preset.ProtoIndex); - if (!presetProto.HasSchema()) { - errStr = "No schema in preset for in-store column table"; - return nullptr; + tableSchema = &presetProto.GetSchema(); + } else { + if (!tableInfo->Description.HasSchema()) { + errors.SetError(NKikimrScheme::StatusSchemeError, "No schema for standalone column table"); + return nullptr; + } + + tableSchema = &tableInfo->Description.GetSchema(); } - tableSchema = &presetProto.GetSchema(); - } else { - if (!tableInfo->Description.HasSchema()) { - errStr = "No schema for standalone column table"; - return nullptr; + THashMap<ui32, TOlapSchema::TColumn> columns; + THashMap<TString, ui32> columnsByName; + for (const auto& col : tableSchema->GetColumns()) { + ui32 id = col.GetId(); + TString name = col.GetName(); + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(col.GetTypeId(), + col.HasTypeInfo() ? &col.GetTypeInfo() : nullptr); + columns[id] = TOlapSchema::TColumn{id, name, typeInfoMod.TypeInfo, Max<ui32>()}; + columnsByName[name] = id; + + // TODO: add checks for compatibility with new schema after we allow such changes } - tableSchema = &tableInfo->Description.GetSchema(); - } + if (AlterRequest.HasAlterTtlSettings()) { + TString errStr; + if (!ValidateTtlSettings(AlterRequest.GetAlterTtlSettings(), columns, columnsByName, errStr)) { + errors.SetError(NKikimrScheme::StatusSchemeError, errStr); + return nullptr; + } - THashMap<ui32, TOlapSchema::TColumn> columns; - THashMap<TString, ui32> columnsByName; - for (const auto& col : tableSchema->GetColumns()) { - ui32 id = col.GetId(); - TString name = col.GetName(); - auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(col.GetTypeId(), - col.HasTypeInfo() ? &col.GetTypeInfo() : nullptr); - columns[id] = TOlapSchema::TColumn{id, name, typeInfoMod.TypeInfo, Max<ui32>()}; - columnsByName[name] = id; - - // TODO: add checks for compatibility with new schema after we allow such changes + *alterData->Description.MutableTtlSettings() = AlterRequest.GetAlterTtlSettings(); + alterData->Description.MutableTtlSettings()->SetVersion(currentTtlVersion + 1); + } + return alterData; } - if (alter.HasAlterTtlSettings()) { - if (!ValidateTtlSettings(alter.GetAlterTtlSettings(), columns, columnsByName, errStr)) { - return nullptr; +private: + bool ParseFromDSRequest(const NKikimrSchemeOp::TTableDescription& dsDescription, NKikimrSchemeOp::TAlterColumnTable& olapDescription, TProposeResponse& errors) const { + olapDescription.SetName(dsDescription.GetName()); + + if (dsDescription.HasTTLSettings()) { + auto& tableTtl = dsDescription.GetTTLSettings(); + NKikimrSchemeOp::TColumnDataLifeCycle* alterTtl = olapDescription.MutableAlterTtlSettings(); + if (tableTtl.HasEnabled()) { + auto& enabled = tableTtl.GetEnabled(); + auto* alterEnabled = alterTtl->MutableEnabled(); + if (enabled.HasColumnName()) { + alterEnabled->SetColumnName(enabled.GetColumnName()); + } + if (enabled.HasExpireAfterSeconds()) { + alterEnabled->SetExpireAfterSeconds(enabled.GetExpireAfterSeconds()); + } + if (enabled.HasColumnUnit()) { + alterEnabled->SetColumnUnit(enabled.GetColumnUnit()); + } + } else if (tableTtl.HasDisabled()) { + alterTtl->MutableDisabled(); + } + if (tableTtl.HasUseTiering()) { + alterTtl->SetUseTiering(tableTtl.GetUseTiering()); + } + } + + for (auto&& dsColumn : dsDescription.GetColumns()) { + NKikimrSchemeOp::TAlterColumnTableSchema* alterSchema = olapDescription.MutableAlterSchema(); + NKikimrSchemeOp::TOlapColumnDescription* olapColumn = alterSchema->AddColumns(); + if (!ParseFromDSRequest(dsColumn, *olapColumn, errors)) { + return false; + } } - *alterData->Description.MutableTtlSettings() = alter.GetAlterTtlSettings(); - alterData->Description.MutableTtlSettings()->SetVersion(currentTtlVersion + 1); + for (auto&& dsColumn : dsDescription.GetDropColumns()) { + NKikimrSchemeOp::TAlterColumnTableSchema* alterSchema = olapDescription.MutableAlterSchema(); + NKikimrSchemeOp::TOlapColumnDescription* olapColumn = alterSchema->AddDropColumns(); + if (!ParseFromDSRequest(dsColumn, *olapColumn, errors)) { + return false; + } + } + return true; } - tableInfo->AlterData = alterData; - return alterData; -} + bool ParseFromDSRequest(const NKikimrSchemeOp::TColumnDescription& dsColumn, NKikimrSchemeOp::TOlapColumnDescription& olapColumn, TProposeResponse& errors) const { + olapColumn.SetName(dsColumn.GetName()); + olapColumn.SetType(dsColumn.GetType()); + if (dsColumn.HasTypeId()) { + olapColumn.SetTypeId(dsColumn.GetTypeId()); + } + if (dsColumn.HasTypeInfo()) { + *olapColumn.MutableTypeInfo() = dsColumn.GetTypeInfo(); + } + if (dsColumn.HasNotNull()) { + olapColumn.SetNotNull(dsColumn.GetNotNull()); + } + if (dsColumn.HasId()) { + olapColumn.SetId(dsColumn.GetId()); + } + if (dsColumn.HasDefaultFromSequence()) { + errors.SetError(NKikimrScheme::StatusInvalidParameter, "DefaultFromSequence not supported"); + return false; + } + if (dsColumn.HasFamilyName() || dsColumn.HasFamily()) { + errors.SetError(NKikimrScheme::StatusInvalidParameter, "FamilyName and Family not supported"); + return false; + } + return true; + } +}; class TConfigureParts: public TSubOperationState { private: @@ -153,8 +226,8 @@ public: TString pathString = path.PathString(); auto tableInfo = context.SS->ColumnTables.TakeVerified(pathId); - TColumnTableInfo::TPtr alterInfo = tableInfo->AlterData; - Y_VERIFY(alterInfo); + TColumnTableInfo::TPtr alterData = tableInfo->AlterData; + Y_VERIFY(alterData); TOlapStoreInfo::TPtr storeInfo; if (auto olapStorePath = path.FindOlapStore()) { @@ -173,25 +246,32 @@ public: auto* alter = tx.MutableAlterTable(); alter->SetPathId(pathId.LocalPathId); - *alter->MutableAlterBody() = *alterInfo->AlterBody; - if (alterInfo->Description.HasSchema()) { - Y_VERIFY(!storeInfo, "Unexpected olap store with schema specified"); - *alter->MutableSchema() = alterInfo->Description.GetSchema(); + *alter->MutableAlterBody() = *alterData->AlterBody; + if (alterData->Description.HasSchema()) { + Y_VERIFY(!storeInfo, + "Unexpected olap store with schema specified"); + *alter->MutableSchema() = alterData->Description.GetSchema(); } - if (alterInfo->Description.HasSchemaPresetId()) { - const ui32 presetId = alterInfo->Description.GetSchemaPresetId(); - Y_VERIFY(storeInfo, "Unexpected schema preset without olap store"); - Y_VERIFY(storeInfo->SchemaPresets.contains(presetId), - "Failed to find schema preset %" PRIu32 " in an olap store", presetId); - auto& preset = storeInfo->SchemaPresets.at(presetId); - size_t presetIndex = preset.ProtoIndex; - *alter->MutableSchemaPreset() = storeInfo->Description.GetSchemaPresets(presetIndex); + if (alterData->Description.HasSchemaPresetId()) { + const ui32 presetId = alterData->Description.GetSchemaPresetId(); + Y_VERIFY(storeInfo, + "Unexpected schema preset without olap store"); + Y_VERIFY(storeInfo->SchemaPresets.contains(presetId), + "Failed to find schema preset %" PRIu32 + " in an olap store", + presetId); + auto &preset = storeInfo->SchemaPresets.at(presetId); + size_t presetIndex = preset.ProtoIndex; + *alter->MutableSchemaPreset() = + storeInfo->Description.GetSchemaPresets(presetIndex); } - if (alterInfo->Description.HasTtlSettings()) { - *alter->MutableTtlSettings() = alterInfo->Description.GetTtlSettings(); + if (alterData->Description.HasTtlSettings()) { + *alter->MutableTtlSettings() = + alterData->Description.GetTtlSettings(); } - if (alterInfo->Description.HasSchemaPresetVersionAdj()) { - alter->SetSchemaPresetVersionAdj(alterInfo->Description.GetSchemaPresetVersionAdj()); + if (alterData->Description.HasSchemaPresetVersionAdj()) { + alter->SetSchemaPresetVersionAdj( + alterData->Description.GetSchemaPresetVersionAdj()); } Y_VERIFY(tx.SerializeToString(&columnShardTxBody)); @@ -417,20 +497,13 @@ public: auto result = MakeHolder<TProposeResponse>(NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), ui64(ssId)); - TString errStr; - NKikimrSchemeOp::TAlterColumnTable alter; - if (Transaction.HasAlterColumnTable()) { - alter = Transaction.GetAlterColumnTable(); - } else if (Transaction.HasAlterTable()) { - alter = ConvertAlter(Transaction.GetAlterTable(), errStr); // from DDL (not known table type) - if (!errStr.empty()) { - result->SetError(NKikimrScheme::StatusSchemeError, errStr); - return result; - } + TTableInfoConstructor schemaConstructor(Transaction); + if (!schemaConstructor.CheckAndPrepare(*result)) { + return result; } - + const TString& parentPathStr = Transaction.GetWorkingDir(); - const TString& name = alter.GetName(); + const TString& name = schemaConstructor.GetTableName(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TAlterColumnTable Propose" @@ -438,16 +511,6 @@ public: << ", opId: " << OperationId << ", at schemeshard: " << ssId); - if (!alter.HasName()) { - result->SetError(NKikimrScheme::StatusInvalidParameter, "No table name in Alter"); - return result; - } - - if (alter.HasAlterSchema()) { - result->SetError(NKikimrScheme::StatusInvalidParameter, "Alter schema is not supported for column tables"); - return result; - } - TPath path = TPath::Resolve(parentPathStr, context.SS).Dive(name); { TPath::TChecker checks = path.Check(); @@ -499,12 +562,13 @@ public: storeInfo = context.SS->OlapStores.at(storePathId); } - TColumnTableInfo::TPtr alterData = ParseParams(path, tableInfo, storeInfo, alter, errStr); + TColumnTableInfo::TPtr alterData = schemaConstructor.BuildTableinfo(tableInfo, storeInfo, *result); if (!alterData) { - result->SetError(NKikimrScheme::StatusSchemeError, errStr); return result; } + tableInfo->AlterData = alterData; + TString errStr; if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; diff --git a/ydb/core/tx/schemeshard/schemeshard_tables_storage.h b/ydb/core/tx/schemeshard/schemeshard_tables_storage.h index e76bad7590a..594be5b446d 100644 --- a/ydb/core/tx/schemeshard/schemeshard_tables_storage.h +++ b/ydb/core/tx/schemeshard/schemeshard_tables_storage.h @@ -63,6 +63,9 @@ public: TColumnTableInfo* operator->() { return Object.Get(); } + const TColumnTableInfo* operator->() const { + return Object.Get(); + } ~TTableCreatedGuard() { Y_VERIFY(Owner.Tables.emplace(PathId, Object).second); Owner.OnAddObject(PathId, Object); diff --git a/ydb/core/tx/schemeshard/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap.cpp index bf2d24cf5ea..ec2649c1b50 100644 --- a/ydb/core/tx/schemeshard/ut_olap.cpp +++ b/ydb/core/tx/schemeshard/ut_olap.cpp @@ -565,7 +565,7 @@ Y_UNIT_TEST_SUITE(TOlap) { AlterSchemaPresets { Name: "default" AlterSchema { - AddColumns { Name: "comment" Type: "Utf8" } + Columns { Name: "comment" Type: "Utf8" } } } )", {NKikimrScheme::StatusInvalidParameter}); |