aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorivanmorozov <ivanmorozov@yandex-team.com>2023-05-20 16:46:52 +0300
committerivanmorozov <ivanmorozov@yandex-team.com>2023-05-20 16:46:52 +0300
commitd27e67d2814d9601d5181c6aabda04e5ba4bc78b (patch)
treeee6353e8d4d43f350ab32c29eaf01b12c9fb753e
parenteef1651eb98ca16cac30f13b6b98736ea2f2899a (diff)
downloadydb-d27e67d2814d9601d5181c6aabda04e5ba4bc78b.tar.gz
use ddl for provide request into schemeshard
-rw-r--r--ydb/core/formats/arrow/compression/diff.h10
-rw-r--r--ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.darwin-x86_64.txt3
-rw-r--r--ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.linux-aarch64.txt3
-rw-r--r--ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.linux-x86_64.txt3
-rw-r--r--ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.windows-x86_64.txt3
-rw-r--r--ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp2
-rw-r--r--ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp40
-rw-r--r--ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.h25
-rw-r--r--ydb/core/protos/flat_scheme_op.proto14
-rw-r--r--ydb/core/tx/columnshard/columnshard__write.cpp2
-rw-r--r--ydb/core/tx/columnshard/engines/reader/read_metadata.h2
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp2
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_olap_types.cpp268
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_olap_types.h67
-rw-r--r--ydb/core/tx/schemeshard/ut_olap.cpp2
15 files changed, 334 insertions, 112 deletions
diff --git a/ydb/core/formats/arrow/compression/diff.h b/ydb/core/formats/arrow/compression/diff.h
index 95cbd68212e..490e5e84938 100644
--- a/ydb/core/formats/arrow/compression/diff.h
+++ b/ydb/core/formats/arrow/compression/diff.h
@@ -12,8 +12,8 @@ namespace NKikimr::NArrow {
class TCompressionDiff {
private:
- YDB_READONLY_DEF(std::optional<arrow::Compression::type>, Codec);
- YDB_READONLY_DEF(std::optional<int>, Level);
+ std::optional<arrow::Compression::type> Codec;
+ std::optional<int> Level;
public:
bool IsEmpty() const {
return !Level && !Codec;
@@ -21,5 +21,11 @@ public:
NKikimrSchemeOp::TCompressionOptions SerializeToProto() const;
bool DeserializeFromProto(const NKikimrSchemeOp::TCompressionOptions& proto);
TConclusionStatus DeserializeFromRequestFeatures(const std::map<TString, TString>& features);
+ const std::optional<arrow::Compression::type>& GetCodec() const {
+ return Codec;
+ }
+ const std::optional<int>& GetLevel() const {
+ return Level;
+ }
};
}
diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.darwin-x86_64.txt b/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.darwin-x86_64.txt
index d725083cc3b..f711deb34e8 100644
--- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.darwin-x86_64.txt
+++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.darwin-x86_64.txt
@@ -15,6 +15,7 @@ target_link_libraries(behaviour-tablestore-operations PUBLIC
contrib-libs-cxxsupp
yutil
services-metadata-manager
+ formats-arrow-compression
ydb-core-protos
)
target_sources(behaviour-tablestore-operations PRIVATE
@@ -29,8 +30,10 @@ target_link_libraries(behaviour-tablestore-operations.global PUBLIC
contrib-libs-cxxsupp
yutil
services-metadata-manager
+ formats-arrow-compression
ydb-core-protos
)
target_sources(behaviour-tablestore-operations.global PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp
)
diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.linux-aarch64.txt b/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.linux-aarch64.txt
index 773b452e339..7c5eb393e2a 100644
--- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.linux-aarch64.txt
+++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.linux-aarch64.txt
@@ -16,6 +16,7 @@ target_link_libraries(behaviour-tablestore-operations PUBLIC
contrib-libs-cxxsupp
yutil
services-metadata-manager
+ formats-arrow-compression
ydb-core-protos
)
target_sources(behaviour-tablestore-operations PRIVATE
@@ -31,8 +32,10 @@ target_link_libraries(behaviour-tablestore-operations.global PUBLIC
contrib-libs-cxxsupp
yutil
services-metadata-manager
+ formats-arrow-compression
ydb-core-protos
)
target_sources(behaviour-tablestore-operations.global PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp
)
diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.linux-x86_64.txt b/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.linux-x86_64.txt
index 773b452e339..7c5eb393e2a 100644
--- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.linux-x86_64.txt
+++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.linux-x86_64.txt
@@ -16,6 +16,7 @@ target_link_libraries(behaviour-tablestore-operations PUBLIC
contrib-libs-cxxsupp
yutil
services-metadata-manager
+ formats-arrow-compression
ydb-core-protos
)
target_sources(behaviour-tablestore-operations PRIVATE
@@ -31,8 +32,10 @@ target_link_libraries(behaviour-tablestore-operations.global PUBLIC
contrib-libs-cxxsupp
yutil
services-metadata-manager
+ formats-arrow-compression
ydb-core-protos
)
target_sources(behaviour-tablestore-operations.global PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp
)
diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.windows-x86_64.txt b/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.windows-x86_64.txt
index d725083cc3b..f711deb34e8 100644
--- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.windows-x86_64.txt
+++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/CMakeLists.windows-x86_64.txt
@@ -15,6 +15,7 @@ target_link_libraries(behaviour-tablestore-operations PUBLIC
contrib-libs-cxxsupp
yutil
services-metadata-manager
+ formats-arrow-compression
ydb-core-protos
)
target_sources(behaviour-tablestore-operations PRIVATE
@@ -29,8 +30,10 @@ target_link_libraries(behaviour-tablestore-operations.global PUBLIC
contrib-libs-cxxsupp
yutil
services-metadata-manager
+ formats-arrow-compression
ydb-core-protos
)
target_sources(behaviour-tablestore-operations.global PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp
)
diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp b/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp
index b7d7b3a1d87..4939aa5c3d0 100644
--- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp
+++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp
@@ -29,7 +29,7 @@ NMetadata::NModifications::TObjectOperatorResult TAddColumnOperation::DoDeserial
void TAddColumnOperation::DoSerializeScheme(NKikimrSchemeOp::TAlterColumnTableSchemaPreset& presetProto) const {
auto schemaData = presetProto.MutableAlterSchema();
- auto column = schemaData->AddColumns();
+ auto column = schemaData->AddAddColumns();
column->SetName(ColumnName);
column->SetType(ColumnType);
column->SetNotNull(NotNull);
diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp
new file mode 100644
index 00000000000..068bae1fc99
--- /dev/null
+++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp
@@ -0,0 +1,40 @@
+#include "alter_column.h"
+
+namespace NKikimr::NKqp::NColumnshard {
+
+NKikimr::NMetadata::NModifications::TObjectOperatorResult TAlterColumnOperation::DoDeserialize(const NYql::TObjectSettingsImpl::TFeatures& features) {
+ {
+ auto it = features.find("NAME");
+ if (it == features.end()) {
+ return NMetadata::NModifications::TObjectOperatorResult("can't find alter parameter NAME");
+ }
+ ColumnName = it->second;
+ }
+ {
+ auto it = features.find("LOW_CARDINALITY");
+ if (it != features.end()) {
+ bool value;
+ if (!TryFromString<bool>(it->second, value)) {
+ return NMetadata::NModifications::TObjectOperatorResult("cannot parse LOW_CARDINALITY as bool");
+ }
+ LowCardinality = value;
+ }
+ }
+ auto result = CompressionDiff.DeserializeFromRequestFeatures(features);
+ if (!result) {
+ return NMetadata::NModifications::TObjectOperatorResult(result.GetErrorMessage());
+ }
+ return NMetadata::NModifications::TObjectOperatorResult(true);
+}
+
+void TAlterColumnOperation::DoSerializeScheme(NKikimrSchemeOp::TAlterColumnTableSchemaPreset& presetProto) const {
+ auto schemaData = presetProto.MutableAlterSchema();
+ auto* column = schemaData->AddAlterColumns();
+ column->SetName(ColumnName);
+ *column->MutableCompression() = CompressionDiff.SerializeToProto();
+ if (LowCardinality) {
+ column->SetLowCardinality(*LowCardinality);
+ }
+}
+
+}
diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.h b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.h
new file mode 100644
index 00000000000..34f1e263806
--- /dev/null
+++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.h
@@ -0,0 +1,25 @@
+#include "abstract.h"
+#include <ydb/core/formats/arrow/compression/diff.h>
+
+namespace NKikimr::NKqp::NColumnshard {
+
+class TAlterColumnOperation : public ITableStoreOperation {
+private:
+ static TString GetTypeName() {
+ return "ALTER_COLUMN";
+ }
+
+ static inline auto Registrator = TFactory::TRegistrator<TAlterColumnOperation>(GetTypeName());
+
+ TString ColumnName;
+
+ NArrow::TCompressionDiff CompressionDiff;
+ std::optional<bool> LowCardinality;
+public:
+ NMetadata::NModifications::TObjectOperatorResult DoDeserialize(const NYql::TObjectSettingsImpl::TFeatures& features) override;
+
+ void DoSerializeScheme(NKikimrSchemeOp::TAlterColumnTableSchemaPreset& presetProto) const override;
+};
+
+}
+
diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto
index e218a7b5119..62435785da5 100644
--- a/ydb/core/protos/flat_scheme_op.proto
+++ b/ydb/core/protos/flat_scheme_op.proto
@@ -385,6 +385,12 @@ message TCompressionOptions {
optional int32 CompressionLevel = 3; // Use default compression level if not set (0 != not set)
}
+message TOlapColumnDiff {
+ optional string Name = 1;
+ optional TCompressionOptions Compression = 2;
+ optional bool LowCardinality = 3;
+}
+
message TOlapColumnDescription {
// This id is auto-generated by schemeshard
optional uint32 Id = 1;
@@ -396,6 +402,9 @@ message TOlapColumnDescription {
optional uint32 TypeId = 4; // TypeId cannot be set explicitly, use Type
optional NKikimrProto.TTypeInfo TypeInfo = 6;
optional bool NotNull = 7;
+
+ optional TCompressionOptions Compression = 8;
+ optional bool LowCardinality = 9;
}
enum EColumnTableEngine {
@@ -434,9 +443,10 @@ message TColumnTableSchema {
}
message TAlterColumnTableSchema {
- repeated TOlapColumnDescription Columns = 1;
- repeated TOlapColumnDescription DropColumns = 6;
+ repeated TOlapColumnDescription AddColumns = 1;
//optional TCompressionOptions DefaultCompression = 5;
+ repeated TOlapColumnDescription DropColumns = 6;
+ repeated TOlapColumnDiff AlterColumns = 7;
}
// Schema presets are used to manage multiple tables with the same schema
diff --git a/ydb/core/tx/columnshard/columnshard__write.cpp b/ydb/core/tx/columnshard/columnshard__write.cpp
index e27ca500ac5..5ccca9c758c 100644
--- a/ydb/core/tx/columnshard/columnshard__write.cpp
+++ b/ydb/core/tx/columnshard/columnshard__write.cpp
@@ -189,7 +189,7 @@ void TColumnShard::Handle(TEvColumnShard::TEvWrite::TPtr& ev, const TActorContex
}
LOG_S_INFO("Write (overload) " << data.size() << " bytes into pathId " << tableId
- << (ShardOverloaded()? " [shard]" : "") << (tableOverload? " [table]" : "")
+ << (ShardOverloaded() ? " [shard]" : "") << (tableOverload? " [table]" : "")
<< " at tablet " << TabletID());
}
diff --git a/ydb/core/tx/columnshard/engines/reader/read_metadata.h b/ydb/core/tx/columnshard/engines/reader/read_metadata.h
index 44aab0cd508..52dac3522d9 100644
--- a/ydb/core/tx/columnshard/engines/reader/read_metadata.h
+++ b/ydb/core/tx/columnshard/engines/reader/read_metadata.h
@@ -163,7 +163,7 @@ public:
}
ISnapshotSchema::TPtr GetSnapshotSchema(const TSnapshot& version) const {
- if (version >=Snapshot){
+ if (version >= Snapshot){
return ResultIndexSchema;
}
return IndexVersions.GetSchema(version);
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp
index 1039847bd08..26f0479ac61 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp
@@ -143,7 +143,7 @@ private:
for (auto&& dsColumn : dsDescription.GetColumns()) {
NKikimrSchemeOp::TAlterColumnTableSchema* alterSchema = olapDescription.MutableAlterSchema();
- NKikimrSchemeOp::TOlapColumnDescription* olapColumn = alterSchema->AddColumns();
+ NKikimrSchemeOp::TOlapColumnDescription* olapColumn = alterSchema->AddAddColumns();
if (!ParseFromDSRequest(dsColumn, *olapColumn, errors)) {
return false;
}
diff --git a/ydb/core/tx/schemeshard/schemeshard_olap_types.cpp b/ydb/core/tx/schemeshard/schemeshard_olap_types.cpp
index a8663a9da67..dfc2e54615a 100644
--- a/ydb/core/tx/schemeshard/schemeshard_olap_types.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_olap_types.cpp
@@ -5,36 +5,16 @@
namespace NKikimr::NSchemeShard {
void TOlapColumnSchema::Serialize(NKikimrSchemeOp::TOlapColumnDescription& columnSchema) const {
+ TBase::Serialize(columnSchema);
columnSchema.SetId(Id);
- columnSchema.SetName(Name);
- columnSchema.SetType(TypeName);
- columnSchema.SetNotNull(NotNullFlag);
-
- auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(Type, "");
- columnSchema.SetTypeId(columnType.TypeId);
- if (columnType.TypeInfo) {
- *columnSchema.MutableTypeInfo() = *columnType.TypeInfo;
- }
}
void TOlapColumnSchema::ParseFromLocalDB(const NKikimrSchemeOp::TOlapColumnDescription& columnSchema) {
+ TBase::ParseFromLocalDB(columnSchema);
Id = columnSchema.GetId();
- Name = columnSchema.GetName();
- TypeName = columnSchema.GetType();
-
- if (columnSchema.HasTypeInfo()) {
- Type = NScheme::TypeInfoModFromProtoColumnType(
- columnSchema.GetTypeId(), &columnSchema.GetTypeInfo())
- .TypeInfo;
- } else {
- Type = NScheme::TypeInfoModFromProtoColumnType(
- columnSchema.GetTypeId(), nullptr)
- .TypeInfo;
- }
- NotNullFlag = columnSchema.GetNotNull();
}
- bool TOlapColumnSchema::ParseFromRequest(const NKikimrSchemeOp::TOlapColumnDescription& columnSchema, IErrorCollector& errors) {
+ bool TOlapColumnAdd::ParseFromRequest(const NKikimrSchemeOp::TOlapColumnDescription& columnSchema, IErrorCollector& errors) {
if (!columnSchema.GetName()) {
errors.AddError("Columns cannot have an empty name");
return false;
@@ -42,6 +22,17 @@ namespace NKikimr::NSchemeShard {
Name = columnSchema.GetName();
NotNullFlag = columnSchema.GetNotNull();
TypeName = columnSchema.GetType();
+ if (columnSchema.HasCompression()) {
+ NArrow::TCompression compression = NArrow::TCompression::Default();
+ if (!compression.DeserializeFromProto(columnSchema.GetCompression())) {
+ errors.AddError("Cannot parse compression info");
+ return false;
+ }
+ Compression = compression;
+ }
+ if (columnSchema.HasLowCardinality()) {
+ LowCardinality = columnSchema.GetLowCardinality();
+ }
if (columnSchema.HasTypeId()) {
errors.AddError(TStringBuilder() << "Cannot set TypeId for column '" << Name << ", use Type");
@@ -73,28 +64,101 @@ namespace NKikimr::NSchemeShard {
return true;
}
+ void TOlapColumnAdd::ParseFromLocalDB(const NKikimrSchemeOp::TOlapColumnDescription& columnSchema) {
+ Name = columnSchema.GetName();
+ TypeName = columnSchema.GetType();
+
+ if (columnSchema.HasTypeInfo()) {
+ Type = NScheme::TypeInfoModFromProtoColumnType(
+ columnSchema.GetTypeId(), &columnSchema.GetTypeInfo())
+ .TypeInfo;
+ } else {
+ Type = NScheme::TypeInfoModFromProtoColumnType(
+ columnSchema.GetTypeId(), nullptr)
+ .TypeInfo;
+ }
+ if (columnSchema.HasCompression()) {
+ NArrow::TCompression compression = NArrow::TCompression::Default();
+ Y_VERIFY(compression.DeserializeFromProto(columnSchema.GetCompression()));
+ Compression = compression;
+ }
+ if (columnSchema.HasLowCardinality()) {
+ LowCardinality = columnSchema.GetLowCardinality();
+ }
+ NotNullFlag = columnSchema.GetNotNull();
+ }
+
+ void TOlapColumnAdd::Serialize(NKikimrSchemeOp::TOlapColumnDescription& columnSchema) const {
+ columnSchema.SetName(Name);
+ columnSchema.SetType(TypeName);
+ columnSchema.SetNotNull(NotNullFlag);
+ if (Compression) {
+ *columnSchema.MutableCompression() = Compression->SerializeToProto();
+ }
+ if (LowCardinality && *LowCardinality) {
+ columnSchema.SetLowCardinality(*LowCardinality);
+ }
+
+ auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(Type, "");
+ columnSchema.SetTypeId(columnType.TypeId);
+ if (columnType.TypeInfo) {
+ *columnSchema.MutableTypeInfo() = *columnType.TypeInfo;
+ }
+ }
+
+ bool TOlapColumnAdd::ApplyDiff(const TOlapColumnDiff& diffColumn, IErrorCollector& errors) {
+ Y_VERIFY(GetName() == diffColumn.GetName());
+ if (!diffColumn.GetCompression().IsEmpty()) {
+ if (!Compression) {
+ Compression = NArrow::TCompression::Default();
+ }
+ if (!Compression->ApplyDiff(diffColumn.GetCompression())) {
+ errors.AddError("Cannot merge compression info");
+ return false;
+ }
+ }
+
+ if (diffColumn.GetLowCardinality()) {
+ LowCardinality = diffColumn.GetLowCardinality();
+ }
+ return true;
+ }
bool TOlapSchemaUpdate::Parse(const NKikimrSchemeOp::TColumnTableSchema& tableSchema, IErrorCollector& errors, bool allowNullKeys) {
if (tableSchema.HasEngine()) {
Engine = tableSchema.GetEngine();
}
- TSet<TString> keyColumnNames;
+ TMap<TString, ui32> keyColumnNames;
for (auto&& pkKey : tableSchema.GetKeyColumnNames()) {
- if (keyColumnNames.contains(pkKey)) {
+ if (!keyColumnNames.emplace(pkKey, keyColumnNames.size()).second) {
errors.AddError(Sprintf("Duplicate key column '%s'", pkKey.c_str()));
return false;
}
- keyColumnNames.emplace(pkKey);
- KeyColumnNames.emplace_back(pkKey);
}
TSet<TString> columnNames;
for (auto& columnSchema : tableSchema.GetColumns()) {
- TOlapColumnSchema column;
+ std::optional<ui32> keyOrder;
+ {
+ auto it = keyColumnNames.find(columnSchema.GetName());
+ if (it != keyColumnNames.end()) {
+ keyOrder = it->second;
+ }
+ }
+
+ TOlapColumnAdd column(keyOrder);
if (!column.ParseFromRequest(columnSchema, errors)) {
return false;
}
+ if (column.GetKeyOrder() && *column.GetKeyOrder() == 0) {
+ if (!TOlapSchema::IsAllowedFirstPkType(column.GetType().GetTypeId())) {
+ errors.AddError(TStringBuilder()
+ << "Type '" << column.GetType().GetTypeId() << "' specified for column '" << column.GetName()
+ << "' is not supported in first PK position");
+ return false;
+ }
+ }
if (columnNames.contains(column.GetName())) {
errors.AddError(Sprintf("Duplicate column '%s'", column.GetName().c_str()));
return false;
@@ -106,53 +170,55 @@ namespace NKikimr::NSchemeShard {
}
}
columnNames.emplace(column.GetName());
- Columns.emplace_back(std::move(column));
+ AddColumns.emplace_back(std::move(column));
}
return true;
}
bool TOlapSchemaUpdate::Parse(const NKikimrSchemeOp::TAlterColumnTableSchema& alterRequest, IErrorCollector& errors) {
- TSet<TString> columnNames;
- for (auto& columnSchema : alterRequest.GetColumns()) {
- TOlapColumnSchema column;
+ TSet<TString> addColumnNames;
+ if (alterRequest.DropColumnsSize()) {
+ errors.AddError("Drop columns method not supported for tablestore and table");
+ return false;
+ }
+
+ for (auto& columnSchema : alterRequest.GetAddColumns()) {
+ TOlapColumnAdd column({});
if (!column.ParseFromRequest(columnSchema, errors)) {
return false;
}
- if (columnNames.contains(column.GetName())) {
- errors.AddError(Sprintf("Duplicate column '%s'", column.GetName().c_str()));
+ if (addColumnNames.contains(column.GetName())) {
+ errors.AddError(Sprintf("column '%s' duplication for add", column.GetName().c_str()));
return false;
}
- if (column.IsNotNull()) {
- errors.AddError("Not null updates not supported");
- return false;
- }
- columnNames.emplace(column.GetName());
- Columns.emplace_back(std::move(column));
- }
- return true;
- }
-
- bool TOlapSchema::Update(const TOlapSchemaUpdate& schemaUpdate, IErrorCollector& errors) {
- if (Columns.empty() && schemaUpdate.GetColumns().empty()) {
- errors.AddError("No columns specified");
- return false;
+ addColumnNames.emplace(column.GetName());
+ AddColumns.emplace_back(std::move(column));
}
- if (KeyColumnIds.empty()) {
- if (schemaUpdate.GetKeyColumnNames().empty()) {
- errors.AddError("No primary key specified");
+ TSet<TString> alterColumnNames;
+ for (auto& columnSchemaDiff: alterRequest.GetAlterColumns()) {
+ TOlapColumnDiff columnDiff;
+ if (!columnDiff.ParseFromRequest(columnSchemaDiff, errors)) {
return false;
}
- } else {
- if (!schemaUpdate.GetKeyColumnNames().empty()) {
- errors.AddError("No primary key updates supported");
+ if (addColumnNames.contains(columnDiff.GetName())) {
+ errors.AddError(Sprintf("column '%s' have to be either add or update", columnDiff.GetName().c_str()));
+ return false;
+ }
+ if (alterColumnNames.contains(columnDiff.GetName())) {
+ errors.AddError(Sprintf("column '%s' duplication for update", columnDiff.GetName().c_str()));
return false;
}
+ alterColumnNames.emplace(columnDiff.GetName());
+ AlterColumns.emplace_back(std::move(columnDiff));
}
+ return true;
+ }
- TMap<TString, ui32> keyIndexes;
- for (ui32 i = 0; i < schemaUpdate.GetKeyColumnNames().size(); ++i) {
- keyIndexes[schemaUpdate.GetKeyColumnNames()[i]] = i;
+ bool TOlapSchema::Update(const TOlapSchemaUpdate& schemaUpdate, IErrorCollector& errors) {
+ if (Columns.empty() && schemaUpdate.GetAddColumns().empty()) {
+ errors.AddError("No add columns specified");
+ return false;
}
if (!HasEngine()) {
@@ -164,43 +230,56 @@ namespace NKikimr::NSchemeShard {
}
}
- for (auto&& column : schemaUpdate.GetColumns()) {
+ const bool hasColumnsBefore = KeyColumnIds.size();
+ std::map<ui32, ui32> orderedKeyColumnIds;
+ for (auto&& column : schemaUpdate.GetAddColumns()) {
if (ColumnsByName.contains(column.GetName())) {
- errors.AddError("No special column updates supported");
+ errors.AddError(Sprintf("column '%s' already exists", column.GetName().data()));
return false;
}
- TOlapColumnSchema newColumn = column;
- newColumn.SetId(NextColumnId);
- ++NextColumnId;
-
- if (keyIndexes.contains(newColumn.GetName())) {
- auto keyOrder = keyIndexes.at(newColumn.GetName());
- if (keyOrder == 0) {
- if (!IsAllowedFirstPkType(newColumn.GetType().GetTypeId())) {
- errors.AddError(TStringBuilder()
- << "Type '" << newColumn.GetType().GetTypeId() << "' specified for column '" << newColumn.GetName()
- << "' is not supported in first PK position");
- return false;
- }
+ if (hasColumnsBefore) {
+ if (column.IsNotNull()) {
+ errors.AddError("Cannot add new not null column currently (not supported yet)");
+ return false;
}
- newColumn.SetKeyOrder(keyOrder);
+ if (column.GetKeyOrder()) {
+ errors.AddError(Sprintf("column '%s' is pk column. its impossible to modify pk", column.GetName().data()));
+ return false;
+ }
+ }
+ TOlapColumnSchema newColumn(column, NextColumnId++);
+ if (newColumn.GetKeyOrder()) {
+ Y_VERIFY(orderedKeyColumnIds.emplace(*newColumn.GetKeyOrder(), newColumn.GetId()).second);
}
- ColumnsByName[newColumn.GetName()] = newColumn.GetId();
- Columns[newColumn.GetId()] = std::move(newColumn);
+
+ Y_VERIFY(ColumnsByName.emplace(newColumn.GetName(), newColumn.GetId()).second);
+ Y_VERIFY(Columns.emplace(newColumn.GetId(), std::move(newColumn)).second);
}
- if (KeyColumnIds.empty()) {
- TVector<ui32> keyColumnIds;
- keyColumnIds.reserve(schemaUpdate.GetKeyColumnNames().size());
- for (auto&& columnName : schemaUpdate.GetKeyColumnNames()) {
- auto it = ColumnsByName.find(columnName);
- if (it == ColumnsByName.end()) {
- errors.AddError("Invalid key column " + columnName);
+ for (auto&& columnDiff : schemaUpdate.GetAlterColumns()) {
+ auto it = ColumnsByName.find(columnDiff.GetName());
+ if (it == ColumnsByName.end()) {
+ errors.AddError(Sprintf("column '%s' not exists for altering", columnDiff.GetName().data()));
+ return false;
+ } else {
+ auto itColumn = Columns.find(it->second);
+ Y_VERIFY(itColumn != Columns.end());
+ TOlapColumnSchema& newColumn = itColumn->second;
+ if (!newColumn.ApplyDiff(columnDiff, errors)) {
return false;
}
- keyColumnIds.push_back(it->second);
}
- KeyColumnIds.swap(keyColumnIds);
+ }
+ if (KeyColumnIds.empty()) {
+ auto it = orderedKeyColumnIds.begin();
+ for (ui32 i = 0; i < orderedKeyColumnIds.size(); ++i, ++it) {
+ KeyColumnIds.emplace_back(it->second);
+ Y_VERIFY(i == it->first);
+ }
+ if (KeyColumnIds.empty()) {
+ errors.AddError("No primary key specified");
+ return false;
+ }
}
++Version;
return true;
@@ -221,17 +300,20 @@ namespace NKikimr::NSchemeShard {
TVector<ui32> keyIds;
keyIds.resize(tableSchema.GetKeyColumnNames().size(), 0);
for (const auto& columnSchema : tableSchema.GetColumns()) {
- TOlapColumnSchema column;
- column.ParseFromLocalDB(columnSchema);
+ std::optional<ui32> keyOrder;
+ if (keyIndexes.contains(columnSchema.GetName())) {
+ keyOrder = keyIndexes.at(columnSchema.GetName());
+ }
- if (keyIndexes.contains(column.GetName())) {
- auto keyOrder = keyIndexes.at(column.GetName());
- column.SetKeyOrder(keyOrder);
- Y_VERIFY(keyOrder < keyIds.size());
- keyIds[keyOrder] = column.GetId();
+ TOlapColumnSchema column(keyOrder);
+ column.ParseFromLocalDB(columnSchema);
+ if (keyOrder) {
+ Y_VERIFY(*keyOrder < keyIds.size());
+ keyIds[*keyOrder] = column.GetId();
}
- ColumnsByName[column.GetName()] = column.GetId();
- Columns[column.GetId()] = std::move(column);
+
+ Y_VERIFY(ColumnsByName.emplace(column.GetName(), column.GetId()).second);
+ Y_VERIFY(Columns.emplace(column.GetId(), std::move(column)).second);
}
KeyColumnIds.swap(keyIds);
}
diff --git a/ydb/core/tx/schemeshard/schemeshard_olap_types.h b/ydb/core/tx/schemeshard/schemeshard_olap_types.h
index 923fe0d1ca8..ce487b7e23c 100644
--- a/ydb/core/tx/schemeshard/schemeshard_olap_types.h
+++ b/ydb/core/tx/schemeshard/schemeshard_olap_types.h
@@ -4,7 +4,8 @@
#include "schemeshard.h"
#include <ydb/library/accessor/accessor.h>
#include <ydb/core/scheme/scheme_types_proto.h>
-
+#include <ydb/core/formats/arrow/compression/object.h>
+#include <ydb/core/formats/arrow/compression/diff.h>
namespace NKikimr::NSchemeShard {
@@ -30,30 +31,76 @@ namespace NKikimr::NSchemeShard {
}
};
- class TOlapColumnSchema {
- YDB_ACCESSOR(ui32, Id, Max<ui32>());
- YDB_ACCESSOR(ui32, KeyOrder, Max<ui32>());
+ class TOlapColumnDiff;
+ class TOlapColumnAdd {
+ private:
+ YDB_READONLY_DEF(std::optional<ui32>, KeyOrder);
YDB_READONLY_DEF(TString, Name);
YDB_READONLY_DEF(TString, TypeName);
YDB_READONLY_DEF(NScheme::TTypeInfo, Type);
YDB_FLAG_ACCESSOR(NotNull, false);
-
+ YDB_READONLY_DEF(std::optional<NArrow::TCompression>, Compression);
+ YDB_READONLY_DEF(std::optional<bool>, LowCardinality);
public:
+ TOlapColumnAdd(const std::optional<ui32>& keyOrder)
+ : KeyOrder(keyOrder)
+ {
+
+ }
+ bool ParseFromRequest(const NKikimrSchemeOp::TOlapColumnDescription& columnSchema, IErrorCollector& errors);
+ void ParseFromLocalDB(const NKikimrSchemeOp::TOlapColumnDescription& columnSchema);
+ void Serialize(NKikimrSchemeOp::TOlapColumnDescription& columnSchema) const;
+ bool ApplyDiff(const TOlapColumnDiff& diffColumn, IErrorCollector& errors);
bool IsKeyColumn() const {
- return KeyOrder != Max<ui32>();
+ return !!KeyOrder;
}
+ };
+
+ class TOlapColumnSchema: public TOlapColumnAdd {
+ private:
+ using TBase = TOlapColumnAdd;
+ YDB_READONLY(ui32, Id, Max<ui32>());
+ public:
+ TOlapColumnSchema(const TOlapColumnAdd& base, const ui32 id)
+ : TBase(base)
+ , Id(id) {
+ }
+ TOlapColumnSchema(const std::optional<ui32>& keyOrder)
+ : TBase(keyOrder) {
+
+ }
void Serialize(NKikimrSchemeOp::TOlapColumnDescription& columnSchema) const;
void ParseFromLocalDB(const NKikimrSchemeOp::TOlapColumnDescription& columnSchema);
- bool ParseFromRequest(const NKikimrSchemeOp::TOlapColumnDescription& columnSchema, IErrorCollector& errors);
+ };
+
+ class TOlapColumnDiff {
+ YDB_READONLY_DEF(TString, Name);
+ YDB_READONLY_DEF(NArrow::TCompressionDiff, Compression);
+ YDB_READONLY_DEF(std::optional<bool>, LowCardinality);
+ public:
+ bool ParseFromRequest(const NKikimrSchemeOp::TOlapColumnDiff& columnSchema, IErrorCollector& errors) {
+ Name = columnSchema.GetName();
+ if (!Name) {
+ errors.AddError("empty field name");
+ return false;
+ }
+ if (!Compression.DeserializeFromProto(columnSchema.GetCompression())) {
+ errors.AddError("cannot parse compression diff from proto");
+ return false;
+ }
+ if (columnSchema.HasLowCardinality()) {
+ LowCardinality = columnSchema.GetLowCardinality();
+ }
+ return true;
+ }
};
class TOlapSchemaUpdate {
YDB_READONLY_OPT(NKikimrSchemeOp::EColumnTableEngine, Engine);
- YDB_READONLY_DEF(TVector<TOlapColumnSchema>, Columns);
- YDB_READONLY_DEF(TVector<TString>, KeyColumnNames);
-
+ YDB_READONLY_DEF(TVector<TOlapColumnAdd>, AddColumns);
+ YDB_READONLY_DEF(TVector<TOlapColumnDiff>, AlterColumns);
public:
bool Parse(const NKikimrSchemeOp::TColumnTableSchema& tableSchema, IErrorCollector& errors, bool allowNullKeys = false);
bool Parse(const NKikimrSchemeOp::TAlterColumnTableSchema& alterRequest, IErrorCollector& errors);
diff --git a/ydb/core/tx/schemeshard/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap.cpp
index 6e286a75372..d750c83e7e8 100644
--- a/ydb/core/tx/schemeshard/ut_olap.cpp
+++ b/ydb/core/tx/schemeshard/ut_olap.cpp
@@ -566,7 +566,7 @@ Y_UNIT_TEST_SUITE(TOlap) {
AlterSchemaPresets {
Name: "default"
AlterSchema {
- Columns { Name: "comment" Type: "Utf8" }
+ AddColumns { Name: "comment" Type: "Utf8" }
}
}
)", {NKikimrScheme::StatusAccepted});