diff options
author | ivanmorozov <ivanmorozov@yandex-team.com> | 2023-05-20 16:46:52 +0300 |
---|---|---|
committer | ivanmorozov <ivanmorozov@yandex-team.com> | 2023-05-20 16:46:52 +0300 |
commit | d27e67d2814d9601d5181c6aabda04e5ba4bc78b (patch) | |
tree | ee6353e8d4d43f350ab32c29eaf01b12c9fb753e | |
parent | eef1651eb98ca16cac30f13b6b98736ea2f2899a (diff) | |
download | ydb-d27e67d2814d9601d5181c6aabda04e5ba4bc78b.tar.gz |
use ddl for provide request into schemeshard
15 files changed, 334 insertions, 112 deletions
diff --git a/ydb/core/formats/arrow/compression/diff.h b/ydb/core/formats/arrow/compression/diff.h index 95cbd68212e..490e5e84938 100644 --- a/ydb/core/formats/arrow/compression/diff.h +++ b/ydb/core/formats/arrow/compression/diff.h @@ -12,8 +12,8 @@ namespace NKikimr::NArrow { class TCompressionDiff { private: - YDB_READONLY_DEF(std::optional<arrow::Compression::type>, Codec); - YDB_READONLY_DEF(std::optional<int>, Level); + std::optional<arrow::Compression::type> Codec; + std::optional<int> Level; public: bool IsEmpty() const { return !Level && !Codec; @@ -21,5 +21,11 @@ public: NKikimrSchemeOp::TCompressionOptions SerializeToProto() const; bool DeserializeFromProto(const NKikimrSchemeOp::TCompressionOptions& proto); TConclusionStatus DeserializeFromRequestFeatures(const std::map<TString, TString>& features); + const std::optional<arrow::Compression::type>& GetCodec() const { + return Codec; + } + const std::optional<int>& GetLevel() const { + return Level; + } }; } diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.darwin-x86_64.txt b/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.darwin-x86_64.txt index d725083cc3b..f711deb34e8 100644 --- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.darwin-x86_64.txt @@ -15,6 +15,7 @@ target_link_libraries(behaviour-tablestore-operations PUBLIC contrib-libs-cxxsupp yutil services-metadata-manager + formats-arrow-compression ydb-core-protos ) target_sources(behaviour-tablestore-operations PRIVATE @@ -29,8 +30,10 @@ target_link_libraries(behaviour-tablestore-operations.global PUBLIC contrib-libs-cxxsupp yutil services-metadata-manager + formats-arrow-compression ydb-core-protos ) target_sources(behaviour-tablestore-operations.global PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp ) diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.linux-aarch64.txt b/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.linux-aarch64.txt index 773b452e339..7c5eb393e2a 100644 --- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.linux-aarch64.txt +++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.linux-aarch64.txt @@ -16,6 +16,7 @@ target_link_libraries(behaviour-tablestore-operations PUBLIC contrib-libs-cxxsupp yutil services-metadata-manager + formats-arrow-compression ydb-core-protos ) target_sources(behaviour-tablestore-operations PRIVATE @@ -31,8 +32,10 @@ target_link_libraries(behaviour-tablestore-operations.global PUBLIC contrib-libs-cxxsupp yutil services-metadata-manager + formats-arrow-compression ydb-core-protos ) target_sources(behaviour-tablestore-operations.global PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp ) diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.linux-x86_64.txt b/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.linux-x86_64.txt index 773b452e339..7c5eb393e2a 100644 --- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.linux-x86_64.txt +++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.linux-x86_64.txt @@ -16,6 +16,7 @@ target_link_libraries(behaviour-tablestore-operations PUBLIC contrib-libs-cxxsupp yutil services-metadata-manager + formats-arrow-compression ydb-core-protos ) target_sources(behaviour-tablestore-operations PRIVATE @@ -31,8 +32,10 @@ target_link_libraries(behaviour-tablestore-operations.global PUBLIC contrib-libs-cxxsupp yutil services-metadata-manager + formats-arrow-compression ydb-core-protos ) target_sources(behaviour-tablestore-operations.global PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp ) diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.windows-x86_64.txt b/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.windows-x86_64.txt index d725083cc3b..f711deb34e8 100644 --- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.windows-x86_64.txt +++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.windows-x86_64.txt @@ -15,6 +15,7 @@ target_link_libraries(behaviour-tablestore-operations PUBLIC contrib-libs-cxxsupp yutil services-metadata-manager + formats-arrow-compression ydb-core-protos ) target_sources(behaviour-tablestore-operations PRIVATE @@ -29,8 +30,10 @@ target_link_libraries(behaviour-tablestore-operations.global PUBLIC contrib-libs-cxxsupp yutil services-metadata-manager + formats-arrow-compression ydb-core-protos ) target_sources(behaviour-tablestore-operations.global PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp ) diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp b/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp index b7d7b3a1d87..4939aa5c3d0 100644 --- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp +++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp @@ -29,7 +29,7 @@ NMetadata::NModifications::TObjectOperatorResult TAddColumnOperation::DoDeserial void TAddColumnOperation::DoSerializeScheme(NKikimrSchemeOp::TAlterColumnTableSchemaPreset& presetProto) const { auto schemaData = presetProto.MutableAlterSchema(); - auto column = schemaData->AddColumns(); + auto column = schemaData->AddAddColumns(); column->SetName(ColumnName); column->SetType(ColumnType); column->SetNotNull(NotNull); diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp new file mode 100644 index 00000000000..068bae1fc99 --- /dev/null +++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp @@ -0,0 +1,40 @@ +#include "alter_column.h" + +namespace NKikimr::NKqp::NColumnshard { + +NKikimr::NMetadata::NModifications::TObjectOperatorResult TAlterColumnOperation::DoDeserialize(const NYql::TObjectSettingsImpl::TFeatures& features) { + { + auto it = features.find("NAME"); + if (it == features.end()) { + return NMetadata::NModifications::TObjectOperatorResult("can't find alter parameter NAME"); + } + ColumnName = it->second; + } + { + auto it = features.find("LOW_CARDINALITY"); + if (it != features.end()) { + bool value; + if (!TryFromString<bool>(it->second, value)) { + return NMetadata::NModifications::TObjectOperatorResult("cannot parse LOW_CARDINALITY as bool"); + } + LowCardinality = value; + } + } + auto result = CompressionDiff.DeserializeFromRequestFeatures(features); + if (!result) { + return NMetadata::NModifications::TObjectOperatorResult(result.GetErrorMessage()); + } + return NMetadata::NModifications::TObjectOperatorResult(true); +} + +void TAlterColumnOperation::DoSerializeScheme(NKikimrSchemeOp::TAlterColumnTableSchemaPreset& presetProto) const { + auto schemaData = presetProto.MutableAlterSchema(); + auto* column = schemaData->AddAlterColumns(); + column->SetName(ColumnName); + *column->MutableCompression() = CompressionDiff.SerializeToProto(); + if (LowCardinality) { + column->SetLowCardinality(*LowCardinality); + } +} + +} diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.h b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.h new file mode 100644 index 00000000000..34f1e263806 --- /dev/null +++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.h @@ -0,0 +1,25 @@ +#include "abstract.h" +#include <ydb/core/formats/arrow/compression/diff.h> + +namespace NKikimr::NKqp::NColumnshard { + +class TAlterColumnOperation : public ITableStoreOperation { +private: + static TString GetTypeName() { + return "ALTER_COLUMN"; + } + + static inline auto Registrator = TFactory::TRegistrator<TAlterColumnOperation>(GetTypeName()); + + TString ColumnName; + + NArrow::TCompressionDiff CompressionDiff; + std::optional<bool> LowCardinality; +public: + NMetadata::NModifications::TObjectOperatorResult DoDeserialize(const NYql::TObjectSettingsImpl::TFeatures& features) override; + + void DoSerializeScheme(NKikimrSchemeOp::TAlterColumnTableSchemaPreset& presetProto) const override; +}; + +} + diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index e218a7b5119..62435785da5 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -385,6 +385,12 @@ message TCompressionOptions { optional int32 CompressionLevel = 3; // Use default compression level if not set (0 != not set) } +message TOlapColumnDiff { + optional string Name = 1; + optional TCompressionOptions Compression = 2; + optional bool LowCardinality = 3; +} + message TOlapColumnDescription { // This id is auto-generated by schemeshard optional uint32 Id = 1; @@ -396,6 +402,9 @@ message TOlapColumnDescription { optional uint32 TypeId = 4; // TypeId cannot be set explicitly, use Type optional NKikimrProto.TTypeInfo TypeInfo = 6; optional bool NotNull = 7; + + optional TCompressionOptions Compression = 8; + optional bool LowCardinality = 9; } enum EColumnTableEngine { @@ -434,9 +443,10 @@ message TColumnTableSchema { } message TAlterColumnTableSchema { - repeated TOlapColumnDescription Columns = 1; - repeated TOlapColumnDescription DropColumns = 6; + repeated TOlapColumnDescription AddColumns = 1; //optional TCompressionOptions DefaultCompression = 5; + repeated TOlapColumnDescription DropColumns = 6; + repeated TOlapColumnDiff AlterColumns = 7; } // Schema presets are used to manage multiple tables with the same schema diff --git a/ydb/core/tx/columnshard/columnshard__write.cpp b/ydb/core/tx/columnshard/columnshard__write.cpp index e27ca500ac5..5ccca9c758c 100644 --- a/ydb/core/tx/columnshard/columnshard__write.cpp +++ b/ydb/core/tx/columnshard/columnshard__write.cpp @@ -189,7 +189,7 @@ void TColumnShard::Handle(TEvColumnShard::TEvWrite::TPtr& ev, const TActorContex } LOG_S_INFO("Write (overload) " << data.size() << " bytes into pathId " << tableId - << (ShardOverloaded()? " [shard]" : "") << (tableOverload? " [table]" : "") + << (ShardOverloaded() ? " [shard]" : "") << (tableOverload? " [table]" : "") << " at tablet " << TabletID()); } diff --git a/ydb/core/tx/columnshard/engines/reader/read_metadata.h b/ydb/core/tx/columnshard/engines/reader/read_metadata.h index 44aab0cd508..52dac3522d9 100644 --- a/ydb/core/tx/columnshard/engines/reader/read_metadata.h +++ b/ydb/core/tx/columnshard/engines/reader/read_metadata.h @@ -163,7 +163,7 @@ public: } ISnapshotSchema::TPtr GetSnapshotSchema(const TSnapshot& version) const { - if (version >=Snapshot){ + if (version >= Snapshot){ return ResultIndexSchema; } return IndexVersions.GetSchema(version); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp index 1039847bd08..26f0479ac61 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp @@ -143,7 +143,7 @@ private: for (auto&& dsColumn : dsDescription.GetColumns()) { NKikimrSchemeOp::TAlterColumnTableSchema* alterSchema = olapDescription.MutableAlterSchema(); - NKikimrSchemeOp::TOlapColumnDescription* olapColumn = alterSchema->AddColumns(); + NKikimrSchemeOp::TOlapColumnDescription* olapColumn = alterSchema->AddAddColumns(); if (!ParseFromDSRequest(dsColumn, *olapColumn, errors)) { return false; } diff --git a/ydb/core/tx/schemeshard/schemeshard_olap_types.cpp b/ydb/core/tx/schemeshard/schemeshard_olap_types.cpp index a8663a9da67..dfc2e54615a 100644 --- a/ydb/core/tx/schemeshard/schemeshard_olap_types.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_olap_types.cpp @@ -5,36 +5,16 @@ namespace NKikimr::NSchemeShard { void TOlapColumnSchema::Serialize(NKikimrSchemeOp::TOlapColumnDescription& columnSchema) const { + TBase::Serialize(columnSchema); columnSchema.SetId(Id); - columnSchema.SetName(Name); - columnSchema.SetType(TypeName); - columnSchema.SetNotNull(NotNullFlag); - - auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(Type, ""); - columnSchema.SetTypeId(columnType.TypeId); - if (columnType.TypeInfo) { - *columnSchema.MutableTypeInfo() = *columnType.TypeInfo; - } } void TOlapColumnSchema::ParseFromLocalDB(const NKikimrSchemeOp::TOlapColumnDescription& columnSchema) { + TBase::ParseFromLocalDB(columnSchema); Id = columnSchema.GetId(); - Name = columnSchema.GetName(); - TypeName = columnSchema.GetType(); - - if (columnSchema.HasTypeInfo()) { - Type = NScheme::TypeInfoModFromProtoColumnType( - columnSchema.GetTypeId(), &columnSchema.GetTypeInfo()) - .TypeInfo; - } else { - Type = NScheme::TypeInfoModFromProtoColumnType( - columnSchema.GetTypeId(), nullptr) - .TypeInfo; - } - NotNullFlag = columnSchema.GetNotNull(); } - bool TOlapColumnSchema::ParseFromRequest(const NKikimrSchemeOp::TOlapColumnDescription& columnSchema, IErrorCollector& errors) { + bool TOlapColumnAdd::ParseFromRequest(const NKikimrSchemeOp::TOlapColumnDescription& columnSchema, IErrorCollector& errors) { if (!columnSchema.GetName()) { errors.AddError("Columns cannot have an empty name"); return false; @@ -42,6 +22,17 @@ namespace NKikimr::NSchemeShard { Name = columnSchema.GetName(); NotNullFlag = columnSchema.GetNotNull(); TypeName = columnSchema.GetType(); + if (columnSchema.HasCompression()) { + NArrow::TCompression compression = NArrow::TCompression::Default(); + if (!compression.DeserializeFromProto(columnSchema.GetCompression())) { + errors.AddError("Cannot parse compression info"); + return false; + } + Compression = compression; + } + if (columnSchema.HasLowCardinality()) { + LowCardinality = columnSchema.GetLowCardinality(); + } if (columnSchema.HasTypeId()) { errors.AddError(TStringBuilder() << "Cannot set TypeId for column '" << Name << ", use Type"); @@ -73,28 +64,101 @@ namespace NKikimr::NSchemeShard { return true; } + void TOlapColumnAdd::ParseFromLocalDB(const NKikimrSchemeOp::TOlapColumnDescription& columnSchema) { + Name = columnSchema.GetName(); + TypeName = columnSchema.GetType(); + + if (columnSchema.HasTypeInfo()) { + Type = NScheme::TypeInfoModFromProtoColumnType( + columnSchema.GetTypeId(), &columnSchema.GetTypeInfo()) + .TypeInfo; + } else { + Type = NScheme::TypeInfoModFromProtoColumnType( + columnSchema.GetTypeId(), nullptr) + .TypeInfo; + } + if (columnSchema.HasCompression()) { + NArrow::TCompression compression = NArrow::TCompression::Default(); + Y_VERIFY(compression.DeserializeFromProto(columnSchema.GetCompression())); + Compression = compression; + } + if (columnSchema.HasLowCardinality()) { + LowCardinality = columnSchema.GetLowCardinality(); + } + NotNullFlag = columnSchema.GetNotNull(); + } + + void TOlapColumnAdd::Serialize(NKikimrSchemeOp::TOlapColumnDescription& columnSchema) const { + columnSchema.SetName(Name); + columnSchema.SetType(TypeName); + columnSchema.SetNotNull(NotNullFlag); + if (Compression) { + *columnSchema.MutableCompression() = Compression->SerializeToProto(); + } + if (LowCardinality && *LowCardinality) { + columnSchema.SetLowCardinality(*LowCardinality); + } + + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(Type, ""); + columnSchema.SetTypeId(columnType.TypeId); + if (columnType.TypeInfo) { + *columnSchema.MutableTypeInfo() = *columnType.TypeInfo; + } + } + + bool TOlapColumnAdd::ApplyDiff(const TOlapColumnDiff& diffColumn, IErrorCollector& errors) { + Y_VERIFY(GetName() == diffColumn.GetName()); + if (!diffColumn.GetCompression().IsEmpty()) { + if (!Compression) { + Compression = NArrow::TCompression::Default(); + } + if (!Compression->ApplyDiff(diffColumn.GetCompression())) { + errors.AddError("Cannot merge compression info"); + return false; + } + } + + if (diffColumn.GetLowCardinality()) { + LowCardinality = diffColumn.GetLowCardinality(); + } + return true; + } bool TOlapSchemaUpdate::Parse(const NKikimrSchemeOp::TColumnTableSchema& tableSchema, IErrorCollector& errors, bool allowNullKeys) { if (tableSchema.HasEngine()) { Engine = tableSchema.GetEngine(); } - TSet<TString> keyColumnNames; + TMap<TString, ui32> keyColumnNames; for (auto&& pkKey : tableSchema.GetKeyColumnNames()) { - if (keyColumnNames.contains(pkKey)) { + if (!keyColumnNames.emplace(pkKey, keyColumnNames.size()).second) { errors.AddError(Sprintf("Duplicate key column '%s'", pkKey.c_str())); return false; } - keyColumnNames.emplace(pkKey); - KeyColumnNames.emplace_back(pkKey); } TSet<TString> columnNames; for (auto& columnSchema : tableSchema.GetColumns()) { - TOlapColumnSchema column; + std::optional<ui32> keyOrder; + { + auto it = keyColumnNames.find(columnSchema.GetName()); + if (it != keyColumnNames.end()) { + keyOrder = it->second; + } + } + + TOlapColumnAdd column(keyOrder); if (!column.ParseFromRequest(columnSchema, errors)) { return false; } + if (column.GetKeyOrder() && *column.GetKeyOrder() == 0) { + if (!TOlapSchema::IsAllowedFirstPkType(column.GetType().GetTypeId())) { + errors.AddError(TStringBuilder() + << "Type '" << column.GetType().GetTypeId() << "' specified for column '" << column.GetName() + << "' is not supported in first PK position"); + return false; + } + } if (columnNames.contains(column.GetName())) { errors.AddError(Sprintf("Duplicate column '%s'", column.GetName().c_str())); return false; @@ -106,53 +170,55 @@ namespace NKikimr::NSchemeShard { } } columnNames.emplace(column.GetName()); - Columns.emplace_back(std::move(column)); + AddColumns.emplace_back(std::move(column)); } return true; } bool TOlapSchemaUpdate::Parse(const NKikimrSchemeOp::TAlterColumnTableSchema& alterRequest, IErrorCollector& errors) { - TSet<TString> columnNames; - for (auto& columnSchema : alterRequest.GetColumns()) { - TOlapColumnSchema column; + TSet<TString> addColumnNames; + if (alterRequest.DropColumnsSize()) { + errors.AddError("Drop columns method not supported for tablestore and table"); + return false; + } + + for (auto& columnSchema : alterRequest.GetAddColumns()) { + TOlapColumnAdd column({}); if (!column.ParseFromRequest(columnSchema, errors)) { return false; } - if (columnNames.contains(column.GetName())) { - errors.AddError(Sprintf("Duplicate column '%s'", column.GetName().c_str())); + if (addColumnNames.contains(column.GetName())) { + errors.AddError(Sprintf("column '%s' duplication for add", column.GetName().c_str())); return false; } - if (column.IsNotNull()) { - errors.AddError("Not null updates not supported"); - return false; - } - columnNames.emplace(column.GetName()); - Columns.emplace_back(std::move(column)); - } - return true; - } - - bool TOlapSchema::Update(const TOlapSchemaUpdate& schemaUpdate, IErrorCollector& errors) { - if (Columns.empty() && schemaUpdate.GetColumns().empty()) { - errors.AddError("No columns specified"); - return false; + addColumnNames.emplace(column.GetName()); + AddColumns.emplace_back(std::move(column)); } - if (KeyColumnIds.empty()) { - if (schemaUpdate.GetKeyColumnNames().empty()) { - errors.AddError("No primary key specified"); + TSet<TString> alterColumnNames; + for (auto& columnSchemaDiff: alterRequest.GetAlterColumns()) { + TOlapColumnDiff columnDiff; + if (!columnDiff.ParseFromRequest(columnSchemaDiff, errors)) { return false; } - } else { - if (!schemaUpdate.GetKeyColumnNames().empty()) { - errors.AddError("No primary key updates supported"); + if (addColumnNames.contains(columnDiff.GetName())) { + errors.AddError(Sprintf("column '%s' have to be either add or update", columnDiff.GetName().c_str())); + return false; + } + if (alterColumnNames.contains(columnDiff.GetName())) { + errors.AddError(Sprintf("column '%s' duplication for update", columnDiff.GetName().c_str())); return false; } + alterColumnNames.emplace(columnDiff.GetName()); + AlterColumns.emplace_back(std::move(columnDiff)); } + return true; + } - TMap<TString, ui32> keyIndexes; - for (ui32 i = 0; i < schemaUpdate.GetKeyColumnNames().size(); ++i) { - keyIndexes[schemaUpdate.GetKeyColumnNames()[i]] = i; + bool TOlapSchema::Update(const TOlapSchemaUpdate& schemaUpdate, IErrorCollector& errors) { + if (Columns.empty() && schemaUpdate.GetAddColumns().empty()) { + errors.AddError("No add columns specified"); + return false; } if (!HasEngine()) { @@ -164,43 +230,56 @@ namespace NKikimr::NSchemeShard { } } - for (auto&& column : schemaUpdate.GetColumns()) { + const bool hasColumnsBefore = KeyColumnIds.size(); + std::map<ui32, ui32> orderedKeyColumnIds; + for (auto&& column : schemaUpdate.GetAddColumns()) { if (ColumnsByName.contains(column.GetName())) { - errors.AddError("No special column updates supported"); + errors.AddError(Sprintf("column '%s' already exists", column.GetName().data())); return false; } - TOlapColumnSchema newColumn = column; - newColumn.SetId(NextColumnId); - ++NextColumnId; - - if (keyIndexes.contains(newColumn.GetName())) { - auto keyOrder = keyIndexes.at(newColumn.GetName()); - if (keyOrder == 0) { - if (!IsAllowedFirstPkType(newColumn.GetType().GetTypeId())) { - errors.AddError(TStringBuilder() - << "Type '" << newColumn.GetType().GetTypeId() << "' specified for column '" << newColumn.GetName() - << "' is not supported in first PK position"); - return false; - } + if (hasColumnsBefore) { + if (column.IsNotNull()) { + errors.AddError("Cannot add new not null column currently (not supported yet)"); + return false; } - newColumn.SetKeyOrder(keyOrder); + if (column.GetKeyOrder()) { + errors.AddError(Sprintf("column '%s' is pk column. its impossible to modify pk", column.GetName().data())); + return false; + } + } + TOlapColumnSchema newColumn(column, NextColumnId++); + if (newColumn.GetKeyOrder()) { + Y_VERIFY(orderedKeyColumnIds.emplace(*newColumn.GetKeyOrder(), newColumn.GetId()).second); } - ColumnsByName[newColumn.GetName()] = newColumn.GetId(); - Columns[newColumn.GetId()] = std::move(newColumn); + + Y_VERIFY(ColumnsByName.emplace(newColumn.GetName(), newColumn.GetId()).second); + Y_VERIFY(Columns.emplace(newColumn.GetId(), std::move(newColumn)).second); } - if (KeyColumnIds.empty()) { - TVector<ui32> keyColumnIds; - keyColumnIds.reserve(schemaUpdate.GetKeyColumnNames().size()); - for (auto&& columnName : schemaUpdate.GetKeyColumnNames()) { - auto it = ColumnsByName.find(columnName); - if (it == ColumnsByName.end()) { - errors.AddError("Invalid key column " + columnName); + for (auto&& columnDiff : schemaUpdate.GetAlterColumns()) { + auto it = ColumnsByName.find(columnDiff.GetName()); + if (it == ColumnsByName.end()) { + errors.AddError(Sprintf("column '%s' not exists for altering", columnDiff.GetName().data())); + return false; + } else { + auto itColumn = Columns.find(it->second); + Y_VERIFY(itColumn != Columns.end()); + TOlapColumnSchema& newColumn = itColumn->second; + if (!newColumn.ApplyDiff(columnDiff, errors)) { return false; } - keyColumnIds.push_back(it->second); } - KeyColumnIds.swap(keyColumnIds); + } + if (KeyColumnIds.empty()) { + auto it = orderedKeyColumnIds.begin(); + for (ui32 i = 0; i < orderedKeyColumnIds.size(); ++i, ++it) { + KeyColumnIds.emplace_back(it->second); + Y_VERIFY(i == it->first); + } + if (KeyColumnIds.empty()) { + errors.AddError("No primary key specified"); + return false; + } } ++Version; return true; @@ -221,17 +300,20 @@ namespace NKikimr::NSchemeShard { TVector<ui32> keyIds; keyIds.resize(tableSchema.GetKeyColumnNames().size(), 0); for (const auto& columnSchema : tableSchema.GetColumns()) { - TOlapColumnSchema column; - column.ParseFromLocalDB(columnSchema); + std::optional<ui32> keyOrder; + if (keyIndexes.contains(columnSchema.GetName())) { + keyOrder = keyIndexes.at(columnSchema.GetName()); + } - if (keyIndexes.contains(column.GetName())) { - auto keyOrder = keyIndexes.at(column.GetName()); - column.SetKeyOrder(keyOrder); - Y_VERIFY(keyOrder < keyIds.size()); - keyIds[keyOrder] = column.GetId(); + TOlapColumnSchema column(keyOrder); + column.ParseFromLocalDB(columnSchema); + if (keyOrder) { + Y_VERIFY(*keyOrder < keyIds.size()); + keyIds[*keyOrder] = column.GetId(); } - ColumnsByName[column.GetName()] = column.GetId(); - Columns[column.GetId()] = std::move(column); + + Y_VERIFY(ColumnsByName.emplace(column.GetName(), column.GetId()).second); + Y_VERIFY(Columns.emplace(column.GetId(), std::move(column)).second); } KeyColumnIds.swap(keyIds); } diff --git a/ydb/core/tx/schemeshard/schemeshard_olap_types.h b/ydb/core/tx/schemeshard/schemeshard_olap_types.h index 923fe0d1ca8..ce487b7e23c 100644 --- a/ydb/core/tx/schemeshard/schemeshard_olap_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_olap_types.h @@ -4,7 +4,8 @@ #include "schemeshard.h" #include <ydb/library/accessor/accessor.h> #include <ydb/core/scheme/scheme_types_proto.h> - +#include <ydb/core/formats/arrow/compression/object.h> +#include <ydb/core/formats/arrow/compression/diff.h> namespace NKikimr::NSchemeShard { @@ -30,30 +31,76 @@ namespace NKikimr::NSchemeShard { } }; - class TOlapColumnSchema { - YDB_ACCESSOR(ui32, Id, Max<ui32>()); - YDB_ACCESSOR(ui32, KeyOrder, Max<ui32>()); + class TOlapColumnDiff; + class TOlapColumnAdd { + private: + YDB_READONLY_DEF(std::optional<ui32>, KeyOrder); YDB_READONLY_DEF(TString, Name); YDB_READONLY_DEF(TString, TypeName); YDB_READONLY_DEF(NScheme::TTypeInfo, Type); YDB_FLAG_ACCESSOR(NotNull, false); - + YDB_READONLY_DEF(std::optional<NArrow::TCompression>, Compression); + YDB_READONLY_DEF(std::optional<bool>, LowCardinality); public: + TOlapColumnAdd(const std::optional<ui32>& keyOrder) + : KeyOrder(keyOrder) + { + + } + bool ParseFromRequest(const NKikimrSchemeOp::TOlapColumnDescription& columnSchema, IErrorCollector& errors); + void ParseFromLocalDB(const NKikimrSchemeOp::TOlapColumnDescription& columnSchema); + void Serialize(NKikimrSchemeOp::TOlapColumnDescription& columnSchema) const; + bool ApplyDiff(const TOlapColumnDiff& diffColumn, IErrorCollector& errors); bool IsKeyColumn() const { - return KeyOrder != Max<ui32>(); + return !!KeyOrder; } + }; + + class TOlapColumnSchema: public TOlapColumnAdd { + private: + using TBase = TOlapColumnAdd; + YDB_READONLY(ui32, Id, Max<ui32>()); + public: + TOlapColumnSchema(const TOlapColumnAdd& base, const ui32 id) + : TBase(base) + , Id(id) { + } + TOlapColumnSchema(const std::optional<ui32>& keyOrder) + : TBase(keyOrder) { + + } void Serialize(NKikimrSchemeOp::TOlapColumnDescription& columnSchema) const; void ParseFromLocalDB(const NKikimrSchemeOp::TOlapColumnDescription& columnSchema); - bool ParseFromRequest(const NKikimrSchemeOp::TOlapColumnDescription& columnSchema, IErrorCollector& errors); + }; + + class TOlapColumnDiff { + YDB_READONLY_DEF(TString, Name); + YDB_READONLY_DEF(NArrow::TCompressionDiff, Compression); + YDB_READONLY_DEF(std::optional<bool>, LowCardinality); + public: + bool ParseFromRequest(const NKikimrSchemeOp::TOlapColumnDiff& columnSchema, IErrorCollector& errors) { + Name = columnSchema.GetName(); + if (!Name) { + errors.AddError("empty field name"); + return false; + } + if (!Compression.DeserializeFromProto(columnSchema.GetCompression())) { + errors.AddError("cannot parse compression diff from proto"); + return false; + } + if (columnSchema.HasLowCardinality()) { + LowCardinality = columnSchema.GetLowCardinality(); + } + return true; + } }; class TOlapSchemaUpdate { YDB_READONLY_OPT(NKikimrSchemeOp::EColumnTableEngine, Engine); - YDB_READONLY_DEF(TVector<TOlapColumnSchema>, Columns); - YDB_READONLY_DEF(TVector<TString>, KeyColumnNames); - + YDB_READONLY_DEF(TVector<TOlapColumnAdd>, AddColumns); + YDB_READONLY_DEF(TVector<TOlapColumnDiff>, AlterColumns); public: bool Parse(const NKikimrSchemeOp::TColumnTableSchema& tableSchema, IErrorCollector& errors, bool allowNullKeys = false); bool Parse(const NKikimrSchemeOp::TAlterColumnTableSchema& alterRequest, IErrorCollector& errors); diff --git a/ydb/core/tx/schemeshard/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap.cpp index 6e286a75372..d750c83e7e8 100644 --- a/ydb/core/tx/schemeshard/ut_olap.cpp +++ b/ydb/core/tx/schemeshard/ut_olap.cpp @@ -566,7 +566,7 @@ Y_UNIT_TEST_SUITE(TOlap) { AlterSchemaPresets { Name: "default" AlterSchema { - Columns { Name: "comment" Type: "Utf8" } + AddColumns { Name: "comment" Type: "Utf8" } } } )", {NKikimrScheme::StatusAccepted}); |