diff options
author | pnv1 <pnv@ydb.tech> | 2022-11-01 15:33:49 +0300 |
---|---|---|
committer | pnv1 <pnv@ydb.tech> | 2022-11-01 15:33:49 +0300 |
commit | ee592643d76623dd06501a1b0ae47193f33b090e (patch) | |
tree | 044006608b65241377fb80184b047529d19c4a0b | |
parent | 4b313388ce70a7bd46d46403b4bcca883527d725 (diff) | |
download | ydb-ee592643d76623dd06501a1b0ae47193f33b090e.tar.gz |
Add TABLESTORE support
-rw-r--r-- | ydb/core/kqp/kqp_ic_gateway.cpp | 191 | ||||
-rw-r--r-- | ydb/core/kqp/provider/yql_kikimr_datasink.cpp | 23 | ||||
-rw-r--r-- | ydb/core/kqp/provider/yql_kikimr_datasource.cpp | 2 | ||||
-rw-r--r-- | ydb/core/kqp/provider/yql_kikimr_exec.cpp | 146 | ||||
-rw-r--r-- | ydb/core/kqp/provider/yql_kikimr_expr_nodes.json | 9 | ||||
-rw-r--r-- | ydb/core/kqp/provider/yql_kikimr_gateway.cpp | 9 | ||||
-rw-r--r-- | ydb/core/kqp/provider/yql_kikimr_gateway.h | 45 | ||||
-rw-r--r-- | ydb/core/kqp/provider/yql_kikimr_provider.cpp | 5 | ||||
-rw-r--r-- | ydb/core/kqp/provider/yql_kikimr_provider.h | 6 | ||||
-rw-r--r-- | ydb/core/kqp/provider/yql_kikimr_type_ann.cpp | 35 | ||||
-rw-r--r-- | ydb/core/kqp/ut/kqp_scheme_ut.cpp | 226 | ||||
-rw-r--r-- | ydb/library/yql/providers/common/provider/yql_provider.cpp | 5 | ||||
-rw-r--r-- | ydb/library/yql/providers/common/provider/yql_provider.h | 1 | ||||
-rw-r--r-- | ydb/library/yql/sql/v1/node.h | 9 | ||||
-rw-r--r-- | ydb/library/yql/sql/v1/query.cpp | 27 | ||||
-rw-r--r-- | ydb/library/yql/sql/v1/sql.cpp | 29 |
16 files changed, 688 insertions, 80 deletions
diff --git a/ydb/core/kqp/kqp_ic_gateway.cpp b/ydb/core/kqp/kqp_ic_gateway.cpp index 9002562850b..33d3d2ceb0c 100644 --- a/ydb/core/kqp/kqp_ic_gateway.cpp +++ b/ydb/core/kqp/kqp_ic_gateway.cpp @@ -1248,7 +1248,9 @@ public: schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateColumnTable); NKikimrSchemeOp::TColumnTableDescription* tableDesc = schemeTx.MutableCreateColumnTable(); - FillCreateColumnTableColumnDesc(*tableDesc, pathPair.second, metadata); + + tableDesc->SetName(pathPair.second); + FillColumnTableSchema(*tableDesc->MutableSchema(), *metadata); if (!FillCreateColumnTableDesc(metadata, *tableDesc, code, error)) { IKqpGateway::TGenericResult errResult; @@ -1337,6 +1339,154 @@ public: } } + TFuture<TGenericResult> AlterColumnTable(const TString& cluster, + const NYql::TAlterColumnTableSettings& settings) override { + using TRequest = TEvTxUserProxy::TEvProposeTransaction; + + try { + if (!CheckCluster(cluster)) { + return InvalidCluster<TGenericResult>(cluster); + } + + std::pair<TString, TString> pathPair; + { + TString error; + if (!GetPathPair(settings.Table, pathPair, error, false)) { + return MakeFuture(ResultFromError<TGenericResult>(error)); + } + } + + auto ev = MakeHolder<TRequest>(); + ev->Record.SetDatabaseName(Database); + if (UserToken) { + ev->Record.SetUserToken(UserToken->Serialized); + } + auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); + schemeTx.SetWorkingDir(pathPair.first); + + schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterColumnTable); + NKikimrSchemeOp::TAlterColumnTable* alter = schemeTx.MutableAlterColumnTable(); + alter->SetName(settings.Table); + + return SendSchemeRequest(ev.Release()); + } + catch (yexception& e) { + return MakeFuture(ResultFromException<TGenericResult>(e)); + } + } + + TFuture<TGenericResult> CreateTableStore(const TString& cluster, + const NYql::TCreateTableStoreSettings& settings) override { + using TRequest = TEvTxUserProxy::TEvProposeTransaction; + + try { + if (!CheckCluster(cluster)) { + return InvalidCluster<TGenericResult>(cluster); + } + + std::pair<TString, TString> pathPair; + { + TString error; + if (!GetPathPair(settings.TableStore, pathPair, error, false)) { + return MakeFuture(ResultFromError<TGenericResult>(error)); + } + } + + auto ev = MakeHolder<TRequest>(); + ev->Record.SetDatabaseName(Database); + if (UserToken) { + ev->Record.SetUserToken(UserToken->Serialized); + } + auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); + schemeTx.SetWorkingDir(pathPair.first); + + schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateColumnStore); + NKikimrSchemeOp::TColumnStoreDescription* storeDesc = schemeTx.MutableCreateColumnStore(); + storeDesc->SetName(pathPair.second); + storeDesc->SetColumnShardCount(settings.ShardsCount); + + NKikimrSchemeOp::TColumnTableSchemaPreset* schemaPreset = storeDesc->AddSchemaPresets(); + schemaPreset->SetName("default"); + FillColumnTableSchema(*schemaPreset->MutableSchema(), settings); + + return SendSchemeRequest(ev.Release()); + } + catch (yexception& e) { + return MakeFuture(ResultFromException<TGenericResult>(e)); + } + } + + TFuture<TGenericResult> AlterTableStore(const TString& cluster, + const NYql::TAlterTableStoreSettings& settings) override { + using TRequest = TEvTxUserProxy::TEvProposeTransaction; + + try { + if (!CheckCluster(cluster)) { + return InvalidCluster<TGenericResult>(cluster); + } + + std::pair<TString, TString> pathPair; + { + TString error; + if (!GetPathPair(settings.TableStore, pathPair, error, false)) { + return MakeFuture(ResultFromError<TGenericResult>(error)); + } + } + + auto ev = MakeHolder<TRequest>(); + ev->Record.SetDatabaseName(Database); + if (UserToken) { + ev->Record.SetUserToken(UserToken->Serialized); + } + auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); + schemeTx.SetWorkingDir(pathPair.first); + + schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterColumnStore); + NKikimrSchemeOp::TAlterColumnStore* alter = schemeTx.MutableAlterColumnStore(); + alter->SetName(pathPair.second); + + return SendSchemeRequest(ev.Release()); + } + catch (yexception& e) { + return MakeFuture(ResultFromException<TGenericResult>(e)); + } + } + + TFuture<TGenericResult> DropTableStore(const TString& cluster, + const NYql::TDropTableStoreSettings& settings) override { + using TRequest = TEvTxUserProxy::TEvProposeTransaction; + + try { + if (!CheckCluster(cluster)) { + return InvalidCluster<TGenericResult>(cluster); + } + + std::pair<TString, TString> pathPair; + { + TString error; + if (!GetPathPair(settings.TableStore, pathPair, error, false)) { + return MakeFuture(ResultFromError<TGenericResult>(error)); + } + } + + auto ev = MakeHolder<TRequest>(); + ev->Record.SetDatabaseName(Database); + if (UserToken) { + ev->Record.SetUserToken(UserToken->Serialized); + } + auto& schemeTx = *ev->Record.MutableTransaction()->MutableModifyScheme(); + schemeTx.SetWorkingDir(pathPair.first); + + schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpDropColumnStore); + NKikimrSchemeOp::TDrop* drop = schemeTx.MutableDrop(); + drop->SetName(pathPair.second); + return SendSchemeRequest(ev.Release()); + } + catch (yexception& e) { + return MakeFuture(ResultFromException<TGenericResult>(e)); + } + } + TFuture<TGenericResult> CreateUser(const TString& cluster, const NYql::TCreateUserSettings& settings) override { using TRequest = TEvTxUserProxy::TEvProposeTransaction; @@ -2304,24 +2454,20 @@ private: } } - static void FillCreateColumnTableColumnDesc(NKikimrSchemeOp::TColumnTableDescription& tableDesc, - const TString& name, NYql::TKikimrTableMetadataPtr metadata) + template <typename T> + static void FillColumnTableSchema(NKikimrSchemeOp::TColumnTableSchema& schema, const T& metadata) { - tableDesc.SetName(name); - - TColumnTableSchema& schema = *tableDesc.MutableSchema(); - - Y_ENSURE(metadata->ColumnOrder.size() == metadata->Columns.size()); - for (const auto& name : metadata->ColumnOrder) { - auto columnIt = metadata->Columns.find(name); - Y_ENSURE(columnIt != metadata->Columns.end()); + Y_ENSURE(metadata.ColumnOrder.size() == metadata.Columns.size()); + for (const auto& name : metadata.ColumnOrder) { + auto columnIt = metadata.Columns.find(name); + Y_ENSURE(columnIt != metadata.Columns.end()); TOlapColumnDescription& columnDesc = *schema.AddColumns(); columnDesc.SetName(columnIt->second.Name); columnDesc.SetType(columnIt->second.Type); } - for (TString& keyColumn : metadata->KeyColumnNames) { + for (const auto& keyColumn : metadata.KeyColumnNames) { schema.AddKeyColumnNames(keyColumn); } } @@ -2736,18 +2882,29 @@ private: static bool FillCreateColumnTableDesc(NYql::TKikimrTableMetadataPtr metadata, NKikimrSchemeOp::TColumnTableDescription& tableDesc, Ydb::StatusIds::StatusCode& code, TString& error) { + tableDesc.SetSchemaPresetName("default"); // TODO: CREATE TABLE without TABLESTORE needs schema + + // TODO: not first PK column + if (metadata->KeyColumnNames.empty()) { + code = Ydb::StatusIds::BAD_REQUEST; + error = TStringBuilder() << "No sharding columns to partition by"; + return false; + } + + auto& hashSharding = *tableDesc.MutableSharding()->MutableHashSharding(); + hashSharding.AddColumns(metadata->KeyColumnNames[0]); if (metadata->TableSettings.PartitionByHashFunction) { - auto& hashSharding = *tableDesc.MutableSharding()->MutableHashSharding(); - if (to_lower(metadata->TableSettings.PartitionByHashFunction.GetRef()) == "modulo_n") { - hashSharding.SetFunction(NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_MODULO_N); - } else if (to_lower(metadata->TableSettings.PartitionByHashFunction.GetRef()) == "cloud_logs") { + if (to_lower(metadata->TableSettings.PartitionByHashFunction.GetRef()) == "cloud_logs") { hashSharding.SetFunction(NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_CLOUD_LOGS); } else { code = Ydb::StatusIds::BAD_REQUEST; - error = TStringBuilder() << "Unknown hash function '" << metadata->TableSettings.PartitionByHashFunction.GetRef() << "' to partition by"; + error = TStringBuilder() << "Unknown hash function '" + << metadata->TableSettings.PartitionByHashFunction.GetRef() << "' to partition by"; return false; } + } else { + hashSharding.SetFunction(NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_MODULO_N); } return true; diff --git a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp index 37f443e1bb8..eb869ee0b45 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp @@ -139,7 +139,10 @@ private: auto mode = settings.Mode.Cast(); if (mode == "drop") { - SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); + auto tableType = settings.TableType.IsValid() + ? GetTableTypeFromString(settings.TableType.Cast()) + : ETableType::Table; // v0 support + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath(), tableType); return TStatus::Ok; } else if ( mode == "upsert" || @@ -190,6 +193,11 @@ private: } auto mode = settings.Mode.Cast(); + + auto tableType = settings.TableType.IsValid() + ? GetTableTypeFromString(settings.TableType.Cast()) + : ETableType::Table; // v0 support + if (mode == "create") { if (!settings.Columns) { ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() @@ -197,7 +205,7 @@ private: return TStatus::Error; } - SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath(), tableType); return TStatus::Ok; } else if (mode == "alter") { if (!settings.AlterActions) { @@ -206,7 +214,7 @@ private: return TStatus::Error; } - SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath(), tableType); return TStatus::Ok; } @@ -494,11 +502,15 @@ public: if (mode == "drop") { YQL_ENSURE(!settings.Columns); + auto tableType = settings.TableType.IsValid() + ? settings.TableType.Cast() + : Build<TCoAtom>(ctx, node->Pos()).Value("table").Done(); // v0 support return Build<TKiDropTable>(ctx, node->Pos()) .World(node->Child(0)) .DataSink(node->Child(1)) .Table().Build(key.GetTablePath()) .Settings(settings.Other) + .TableType(tableType) .Done() .Ptr(); } else if (mode == "update") { @@ -537,6 +549,9 @@ public: case TKikimrKey::Type::TableScheme: { NCommon::TWriteTableSettings settings = NCommon::ParseWriteTableSettings(TExprList(node->Child(4)), ctx); YQL_ENSURE(settings.Mode); + auto tableType = settings.TableType.IsValid() + ? settings.TableType.Cast() + : Build<TCoAtom>(ctx, node->Pos()).Value("table").Done(); // v0 support auto mode = settings.Mode.Cast(); if (mode == "create") { YQL_ENSURE(settings.Columns); @@ -563,6 +578,7 @@ public: .PartitionBy(settings.PartitionBy.Cast()) .ColumnFamilies(settings.ColumnFamilies.Cast()) .TableSettings(settings.TableSettings.Cast()) + .TableType(tableType) .Done() .Ptr(); } else if (mode == "alter") { @@ -581,6 +597,7 @@ public: .DataSink(node->Child(1)) .Table().Build(key.GetTablePath()) .Actions(settings.AlterActions.Cast()) + .TableType(tableType) .Done() .Ptr(); } else { diff --git a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp index e8da991e71d..c94eb2b2093 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp @@ -103,7 +103,7 @@ public: const TString& tableName = it.first.second; TKikimrTableDescription& table = SessionCtx->Tables().GetTable(clusterName, tableName); - if (table.Metadata) { + if (table.Metadata || table.GetTableType() != ETableType::Table) { continue; } diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp index c935740d52b..15c679f52d6 100644 --- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp @@ -127,6 +127,64 @@ namespace { } return dropGroupSettings; } + + TCreateTableStoreSettings ParseCreateTableStoreSettings(TKiCreateTable create, const TTableSettings& settings) { + TCreateTableStoreSettings out; + out.TableStore = TString(create.Table()); + out.ShardsCount = settings.MinPartitions ? *settings.MinPartitions : 0; + + for (auto atom : create.PrimaryKey()) { + out.KeyColumnNames.emplace_back(atom.Value()); + } + + for (auto item : create.Columns()) { + auto columnTuple = item.Cast<TExprList>(); + auto nameNode = columnTuple.Item(0).Cast<TCoAtom>(); + auto typeNode = columnTuple.Item(1); + + auto columnName = TString(nameNode.Value()); + auto columnType = typeNode.Ref().GetTypeAnn(); + YQL_ENSURE(columnType && columnType->GetKind() == ETypeAnnotationKind::Type); + + auto type = columnType->Cast<TTypeExprType>()->GetType(); + auto notNull = type->GetKind() != ETypeAnnotationKind::Optional; + auto actualType = notNull ? type : type->Cast<TOptionalExprType>()->GetItemType(); + auto dataType = actualType->Cast<TDataExprType>(); + + TKikimrColumnMetadata columnMeta; + columnMeta.Name = columnName; + columnMeta.Type = dataType->GetName(); + columnMeta.NotNull = notNull; + + out.ColumnOrder.push_back(columnName); + out.Columns.insert(std::make_pair(columnName, columnMeta)); + } +#if 0 // TODO + for (const auto& index : create.Indexes()) { + TIndexDescription indexDesc; + out.Indexes.push_back(indexDesc); + } +#endif + return out; + } + + TAlterTableStoreSettings ParseAlterTableStoreSettings(TKiAlterTable alter) { + return TAlterTableStoreSettings{ + .TableStore = TString(alter.Table()) + }; + } + + TDropTableStoreSettings ParseDropTableStoreSettings(TKiDropTable drop) { + return TDropTableStoreSettings{ + .TableStore = TString(drop.Table()) + }; + } + + TAlterColumnTableSettings ParseAlterColumnTableSettings(TKiAlterTable alter) { + return TAlterColumnTableSettings{ + .Table = TString(alter.Table()) + }; + } } class TKiSinkPlanInfoTransformer : public TGraphTransformerBase { @@ -469,24 +527,39 @@ public: auto cluster = TString(maybeCreate.Cast().DataSink().Cluster()); auto& table = SessionCtx->Tables().GetTable(cluster, TString(maybeCreate.Cast().Table())); - if (!ApplyDdlOperation(cluster, input->Pos(), table.Metadata->Name, TYdbOperation::CreateTable, ctx)) { + bool isTableStore = (table.Metadata->TableType == ETableType::TableStore); + + if (!isTableStore && !ApplyDdlOperation(cluster, input->Pos(), table.Metadata->Name, TYdbOperation::CreateTable, ctx)) { return SyncError(); } - bool prepareOnly = SessionCtx->Query().PrepareOnly; - auto future = prepareOnly ? CreateDummySuccess() : ( - table.Metadata->TableSettings.StoreType - && to_lower(table.Metadata->TableSettings.StoreType.GetRef()) == "column" - ? Gateway->CreateColumnTable(table.Metadata, true) - : Gateway->CreateTable(table.Metadata, true) - ); + NThreading::TFuture<IKikimrGateway::TGenericResult> future; + if (SessionCtx->Query().PrepareOnly) { + future = CreateDummySuccess(); + } else { + bool isColumn = (table.Metadata->StoreType == EStoreType::Column); + + if (isTableStore) { + if (!isColumn) { + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), + TStringBuilder() << "TABLESTORE with not COLUMN store")); + return SyncError(); + } + future = Gateway->CreateTableStore(cluster, + ParseCreateTableStoreSettings(maybeCreate.Cast(), table.Metadata->TableSettings)); + } else if (isColumn) { + future = Gateway->CreateColumnTable(table.Metadata, true); + } else { + future = Gateway->CreateTable(table.Metadata, true); + } + } return WrapFuture(future, [](const IKikimrGateway::TGenericResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { Y_UNUSED(res); auto resultNode = ctx.NewWorld(input->Pos()); return resultNode; - }, "Executing CREATE TABLE"); + }, isTableStore ? "Executing CREATE TABLESTORE" : "Executing CREATE TABLE"); } if (auto maybeDrop = TMaybeNode<TKiDropTable>(input)) { @@ -498,23 +571,42 @@ public: if (requireStatus.Level != TStatus::Ok) { return SyncStatus(requireStatus); } - auto cluster = TString(maybeDrop.Cast().DataSink().Cluster()); - auto& table = SessionCtx->Tables().GetTable(cluster, TString(maybeDrop.Cast().Table())); - - if (!ApplyDdlOperation(cluster, input->Pos(), table.Metadata->Name, TYdbOperation::DropTable, ctx)) { - return SyncError(); + TString tableName = TString(maybeDrop.Cast().Table()); + auto& table = SessionCtx->Tables().GetTable(cluster, tableName); + auto tableType = TString(maybeDrop.Cast().TableType()); + bool isTableStore; + + switch (GetTableTypeFromString(tableType)) { + case ETableType::Table: + if (!ApplyDdlOperation(cluster, input->Pos(), tableName, TYdbOperation::DropTable, ctx)) { + return SyncError(); + } + isTableStore = false; + break; + case ETableType::TableStore: + isTableStore = true; + break; + case ETableType::Unknown: + default: + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Unsupported table type " << tableType)); + return SyncError(); } bool prepareOnly = SessionCtx->Query().PrepareOnly; - auto future = prepareOnly ? CreateDummySuccess() : Gateway->DropTable(table.Metadata->Cluster, table.Metadata->Name); + auto future = prepareOnly ? CreateDummySuccess() : ( + isTableStore + ? Gateway->DropTableStore(cluster, ParseDropTableStoreSettings(maybeDrop.Cast())) + : Gateway->DropTable(table.Metadata->Cluster, table.Metadata->Name) + ); + return WrapFuture(future, [](const IKikimrGateway::TGenericResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { Y_UNUSED(res); auto resultNode = ctx.NewWorld(input->Pos()); return resultNode; - }); + }, isTableStore ? "Executing DROP TABLESTORE" : "Executing DROP TABLE"); input->SetState(TExprNode::EState::ExecutionComplete); input->SetResult(ctx.NewWorld(input->Pos())); @@ -913,8 +1005,26 @@ public: } } - bool prepareOnly = SessionCtx->Query().PrepareOnly; - auto future = prepareOnly ? CreateDummySuccess() : Gateway->AlterTable(std::move(alterTableRequest), cluster); + NThreading::TFuture<IKikimrGateway::TGenericResult> future; + if (SessionCtx->Query().PrepareOnly) { + future = CreateDummySuccess(); + } else { + bool isTableStore = (table.Metadata->TableType == ETableType::TableStore); + bool isColumn = (table.Metadata->StoreType == EStoreType::Column); + + if (isTableStore) { + if (!isColumn) { + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), + TStringBuilder() << "TABLESTORE with not COLUMN store")); + return SyncError(); + } + future = Gateway->AlterTableStore(cluster, ParseAlterTableStoreSettings(maybeAlter.Cast())); + } else if (isColumn) { + future = Gateway->AlterColumnTable(cluster, ParseAlterColumnTableSettings(maybeAlter.Cast())); + } else { + future = Gateway->AlterTable(std::move(alterTableRequest), cluster); + } + } return WrapFuture(future, [](const IKikimrGateway::TGenericResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { diff --git a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json index c54d82a2cac..d801df3a317 100644 --- a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json +++ b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json @@ -117,7 +117,8 @@ {"Index": 7, "Name": "PartitionBy", "Type": "TCoAtomList"}, {"Index": 8, "Name": "ColumnFamilies", "Type": "TExprList"}, {"Index": 9, "Name": "TableSettings", "Type": "TCoNameValueTupleList"}, - {"Index": 10, "Name": "Changefeeds", "Type": "TCoChangefeedList"} + {"Index": 10, "Name": "Changefeeds", "Type": "TCoChangefeedList"}, + {"Index": 11, "Name": "TableType", "Type": "TCoAtom"} ] }, { @@ -128,7 +129,8 @@ {"Index": 0, "Name": "World", "Type": "TExprBase"}, {"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"}, {"Index": 2, "Name": "Table", "Type": "TCoAtom"}, - {"Index": 3, "Name": "Actions", "Type": "TCoNameValueTupleList"} + {"Index": 3, "Name": "Actions", "Type": "TCoNameValueTupleList"}, + {"Index": 4, "Name": "TableType", "Type": "TCoAtom"} ] }, { @@ -139,7 +141,8 @@ {"Index": 0, "Name": "World", "Type": "TExprBase"}, {"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"}, {"Index": 2, "Name": "Table", "Type": "TCoAtom"}, - {"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"} + {"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"}, + {"Index": 4, "Name": "TableType", "Type": "TCoAtom"} ] }, { diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp index d60744455b5..638838995b6 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp @@ -370,4 +370,13 @@ Ydb::FeatureFlag::Status GetFlagValue(const TMaybe<bool>& value) { : Ydb::FeatureFlag::DISABLED; } +ETableType GetTableTypeFromString(const TStringBuf& tableType) { + if (tableType == "table") { + return ETableType::Table; + } else if (tableType == "tableStore") { + return ETableType::TableStore; + } + return ETableType::Unknown; +} + } // namespace NYql diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index d536091be73..6791ae8f58f 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -174,7 +174,6 @@ struct TTableSettings { TMaybe<TString> KeyBloomFilter; TMaybe<TString> ReadReplicasSettings; TResetableSetting<TTtlSettings, void> TtlSettings; - TMaybe<TString> StoreType; TMaybe<TString> PartitionByHashFunction; bool IsSet() const; @@ -279,6 +278,19 @@ enum class EKikimrTableKind : ui32 { Olap = 3 }; +enum class ETableType : ui32 { + Unknown = 0, + Table = 1, + TableStore = 2 +}; + +ETableType GetTableTypeFromString(const TStringBuf& tableType); + +enum class EStoreType : ui32 { + Row = 0, + Column = 1 +}; + struct TKikimrTableMetadata : public TThrRefBase { bool DoesExist = false; TString Cluster; @@ -288,6 +300,8 @@ struct TKikimrTableMetadata : public TThrRefBase { ui64 SchemaVersion = 0; THashMap<TString, TString> Attributes; EKikimrTableKind Kind = EKikimrTableKind::Unspecified; + ETableType TableType = ETableType::Table; + EStoreType StoreType = EStoreType::Row; ui64 RecordsCount = 0; ui64 DataSize = 0; @@ -467,6 +481,27 @@ struct TDropGroupSettings { bool Force = false; }; +struct TAlterColumnTableSettings { + TString Table; +}; + +struct TCreateTableStoreSettings { + TString TableStore; + ui32 ShardsCount = 0; + TMap<TString, TKikimrColumnMetadata> Columns; + TVector<TString> KeyColumnNames; + TVector<TString> ColumnOrder; + TVector<TIndexDescription> Indexes; +}; + +struct TAlterTableStoreSettings { + TString TableStore; +}; + +struct TDropTableStoreSettings { + TString TableStore; +}; + struct TKikimrListPathItem { TKikimrListPathItem(TString name, bool isDirectory) { Name = name; @@ -626,6 +661,14 @@ public: virtual NThreading::TFuture<TGenericResult> CreateColumnTable(TKikimrTableMetadataPtr metadata, bool createDir) = 0; + virtual NThreading::TFuture<TGenericResult> AlterColumnTable(const TString& cluster, const TAlterColumnTableSettings& settings) = 0; + + virtual NThreading::TFuture<TGenericResult> CreateTableStore(const TString& cluster, const TCreateTableStoreSettings& settings) = 0; + + virtual NThreading::TFuture<TGenericResult> AlterTableStore(const TString& cluster, const TAlterTableStoreSettings& settings) = 0; + + virtual NThreading::TFuture<TGenericResult> DropTableStore(const TString& cluster, const TDropTableStoreSettings& settings) = 0; + virtual TVector<TString> GetCollectedSchemeData() = 0; public: diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.cpp b/ydb/core/kqp/provider/yql_kikimr_provider.cpp index a60d91fef7a..df940c05035 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_provider.cpp @@ -121,7 +121,7 @@ const TKikimrTableDescription* TKikimrTablesData::EnsureTableExists(const TStrin const TString& table, TPositionHandle pos, TExprContext& ctx) const { auto desc = Tables.FindPtr(std::make_pair(cluster, table)); - if (desc && desc->DoesExist()) { + if (desc && (desc->GetTableType() != ETableType::Table || desc->DoesExist())) { return desc; } @@ -132,7 +132,7 @@ const TKikimrTableDescription* TKikimrTablesData::EnsureTableExists(const TStrin return nullptr; } -TKikimrTableDescription& TKikimrTablesData::GetOrAddTable(const TString& cluster, const TString& database, const TString& table) { +TKikimrTableDescription& TKikimrTablesData::GetOrAddTable(const TString& cluster, const TString& database, const TString& table, ETableType tableType) { if (!Tables.FindPtr(std::make_pair(cluster, table))) { auto& desc = Tables[std::make_pair(cluster, table)]; @@ -141,6 +141,7 @@ TKikimrTableDescription& TKikimrTablesData::GetOrAddTable(const TString& cluster if (NKikimr::TrySplitPathByDb(table, database, pathPair, error)) { desc.RelativePath = pathPair.second; } + desc.SetTableType(tableType); return desc; } diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.h b/ydb/core/kqp/provider/yql_kikimr_provider.h index 4e0b0c5f807..a5cc6e66fb9 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.h +++ b/ydb/core/kqp/provider/yql_kikimr_provider.h @@ -216,10 +216,13 @@ public: void RequireStats() { NeedsStats = true; } bool GetNeedsStats() const { return NeedsStats; } + ETableType GetTableType() const { return TableType; } + void SetTableType(ETableType tableType) { TableType = tableType; } private: THashMap<TString, const TTypeAnnotationNode*> ColumnTypes; bool NeedsStats = false; + ETableType TableType; }; class TKikimrTablesData : public TThrRefBase { @@ -228,7 +231,8 @@ public: TKikimrTablesData(const TKikimrTablesData&) = delete; TKikimrTablesData& operator=(const TKikimrTablesData&) = delete; - TKikimrTableDescription& GetOrAddTable(const TString& cluster, const TString& database, const TString& table); + TKikimrTableDescription& GetOrAddTable(const TString& cluster, const TString& database, const TString& table, + ETableType tableType = ETableType::Table); TKikimrTableDescription& GetTable(const TString& cluster, const TString& table); const TKikimrTableDescription* EnsureTableExists(const TString& cluster, const TString& table, diff --git a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp index 757a9586436..8e84a6e969b 100644 --- a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp @@ -557,6 +557,7 @@ private: virtual TStatus HandleCreateTable(TKiCreateTable create, TExprContext& ctx) override { TString cluster = TString(create.DataSink().Cluster()); TString table = TString(create.Table()); + TString tableType = TString(create.TableType()); auto columnTypeError = GetColumnTypeErrorFn(ctx); @@ -564,6 +565,14 @@ private: meta->DoesExist = true; meta->ColumnOrder.reserve(create.Columns().Size()); + auto tableTypeEnum = GetTableTypeFromString(tableType); + if (tableTypeEnum == ETableType::Unknown) { + ctx.AddError(TIssue(ctx.GetPosition(create.Pos()), TStringBuilder() + << "Unknown table type: " << tableType << ".")); + return TStatus::Error; + } + meta->TableType = tableTypeEnum; + for (auto atom : create.PrimaryKey()) { meta->KeyColumnNames.emplace_back(atom.Value()); } @@ -821,7 +830,10 @@ private: "Can't reset TTL settings")); return TStatus::Error; } else if (name == "storeType") { - meta->TableSettings.StoreType = TString(setting.Value().Cast<TCoAtom>().Value()); + TMaybe<TString> storeType = TString(setting.Value().Cast<TCoAtom>().Value()); + if (storeType && to_lower(storeType.GetRef()) == "column") { + meta->StoreType = EStoreType::Column; + } } else if (name == "partitionByHashFunction") { meta->TableSettings.PartitionByHashFunction = TString( setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() @@ -833,12 +845,17 @@ private: } } - if (!EnsureModifyPermissions(cluster, table, create.Pos(), ctx)) { + if (meta->TableType == ETableType::TableStore && meta->StoreType != EStoreType::Column) { + ctx.AddError(TIssue(ctx.GetPosition(create.Pos()), + TStringBuilder() << "TABLESTORE recuires STORE = COLUMN setting now")); return TStatus::Error; } + if (!EnsureModifyPermissions(cluster, table, create.Pos(), ctx)) { + return TStatus::Error; + } auto& tableDesc = SessionCtx->Tables().GetTable(cluster, table); - if (tableDesc.DoesExist() && !tableDesc.Metadata->IsSameTable(*meta)) { + if (meta->TableType == ETableType::Table && tableDesc.DoesExist() && !tableDesc.Metadata->IsSameTable(*meta)) { ctx.AddError(TIssue(ctx.GetPosition(create.Pos()), TStringBuilder() << "Table name conflict: " << NCommon::FullTableName(cluster, table) << " is used to reference multiple tables.")); @@ -859,12 +876,14 @@ private: return TStatus::Error; } - if (!EnsureModifyPermissions(table->Metadata->Cluster, table->Metadata->Name, node.Pos(), ctx)) { - return TStatus::Error; - } + if (table->GetTableType() == ETableType::Table) { + if (!EnsureModifyPermissions(table->Metadata->Cluster, table->Metadata->Name, node.Pos(), ctx)) { + return TStatus::Error; + } - if (!CheckDocApiModifiation(*table->Metadata, node.Pos(), ctx)) { - return TStatus::Error; + if (!CheckDocApiModifiation(*table->Metadata, node.Pos(), ctx)) { + return TStatus::Error; + } } node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn()); diff --git a/ydb/core/kqp/ut/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/kqp_scheme_ut.cpp index 72db5475d51..d20202bf59f 100644 --- a/ydb/core/kqp/ut/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/kqp_scheme_ut.cpp @@ -2878,41 +2878,251 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } } - Y_UNIT_TEST(CreateTableStoreSimple) { + Y_UNIT_TEST(CreateAlterDropTableStore) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - TString tableStoreName = "/Root/ColumnTableTest"; + TString tableStoreName = "/Root/TableStoreTest"; auto query = TStringBuilder() << R"( --!syntax_v1 - CREATE TABLE `)" << tableStoreName << R"(` ( + CREATE TABLESTORE `)" << tableStoreName << R"(` ( Key Uint64, Value1 String, PRIMARY KEY (Key) ) WITH ( STORE = COLUMN, - PARTITION_BY_HASH_FUNCTION = "asd" + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10 );)"; auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + +#if 0 // TODO + auto query2 = TStringBuilder() << R"( + --!syntax_v1 + ALTER TABLESTORE `)" << tableStoreName << R"(` + SET (AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10) + ;)"; + result = session.ExecuteSchemeQuery(query2).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); +#endif + auto query3 = TStringBuilder() << R"( + --!syntax_v1 + DROP TABLESTORE `)" << tableStoreName << R"(`;)"; + result = session.ExecuteSchemeQuery(query3).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + +#if 0 // TODO + Y_UNIT_TEST(CreateDropInheritedColumnTable) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableTest"; + TString tableStoreName = "/Root/TableStoreTest"; + TString columnTableName = "/Root/TableStoreTest/ColumnTableTest"; + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value1 String, + PRIMARY KEY (Key) + ) + PARTITION BY HASH(Key) + WITH ( + STORE = ROW, + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10 + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto query2 = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLESTORE `)" << tableStoreName << R"(` ( + Key Uint64, + Value1 String, + PRIMARY KEY (Key) + ) + WITH ( + STORE = COLUMN, + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10 + );)"; + result = session.ExecuteSchemeQuery(query2).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto query3 = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << columnTableName << R"(` () + INHERITS `)" << tableName << R"(` + PARTITION BY HASH(Key) + WITH ( + STORE = COLUMN, + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 5 + ) + TABLESTORE `)" << tableStoreName << R"(`;)"; + result = session.ExecuteSchemeQuery(query3).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto query4 = TStringBuilder() << R"( + --!syntax_v1 + DROP TABLE `)" << tableName << R"(`;)"; + result = session.ExecuteSchemeQuery(query4).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto query5 = TStringBuilder() << R"( + --!syntax_v1 + DROP TABLE `)" << columnTableName << R"(`;)"; + result = session.ExecuteSchemeQuery(query5).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } +#endif + + Y_UNIT_TEST(CreateTableStoreNegative) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableStoreName = "/Root/TableStoreTest"; + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLESTORE `)" << tableStoreName << R"(` ( + Key Uint64, + Value1 String, + PRIMARY KEY (Key) + ) + WITH ( + STORE = ROW, + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10 + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + + auto query2 = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLESTORE `)" << tableStoreName << R"(` ( + Key Uint64, + Value1 String, + PRIMARY KEY (Key) + ) + WITH ( + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10 + );)"; + result = session.ExecuteSchemeQuery(query2).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + + auto query3 = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLESTORE `)" << tableStoreName << R"(` ( + Key Uint64, + Value1 String, + PRIMARY KEY (Key) + ) + WITH ( + STORE = COLUMN + );)"; + result = session.ExecuteSchemeQuery(query3).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + } + + Y_UNIT_TEST(CreateAlterDropColumnTableInStore) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableStoreName = "/Root/TableStoreTest"; + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLESTORE `)" << tableStoreName << R"(` ( + Key Uint64, + Value1 String, + PRIMARY KEY (Key) + ) + WITH ( + STORE = COLUMN, + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10 + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + +#if 0 // TODO + TString tableName = "/Root/TableStoreTest/ColumnTableTest"; + auto query2 = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value1 String, + PRIMARY KEY (Key) + ) + PARTITION BY HASH (Key) + WITH ( + STORE = COLUMN, + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10 + );)"; + result = session.ExecuteSchemeQuery(query2).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto query3 = TStringBuilder() << R"( + --!syntax_v1 + ALTER TABLE `)" << tableName << R"(`;)"; + result = session.ExecuteSchemeQuery(query3).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto query4 = TStringBuilder() << R"( + --!syntax_v1 + DROP TABLE `)" << tableName << R"(`;)"; + result = session.ExecuteSchemeQuery(query4).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); +#endif + + auto query5 = TStringBuilder() << R"( + --!syntax_v1 + DROP TABLESTORE `)" << tableStoreName << R"(`;)"; + result = session.ExecuteSchemeQuery(query5).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } - Y_UNIT_TEST(CreateTableStoreWithWrongFunction) { +#if 0 // TODO + Y_UNIT_TEST(CreateDropColumnTable) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - TString tableStoreName = "/Root/ColumnTableTest"; + TString tableName = "/Root/ColumnTableTest"; auto query = TStringBuilder() << R"( --!syntax_v1 - CREATE TABLE `)" << tableStoreName << R"(` ( + CREATE TABLE `)" << tableName << R"(` ( + Key Uint64, + Value1 String, + PRIMARY KEY (Key) + ) + PARTITION BY HASH(Key) + WITH ( + STORE = COLUMN, + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10 + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto query2 = TStringBuilder() << R"( + --!syntax_v1 + DROP TABLE `)" << tableName << R"(`;)"; + result = session.ExecuteSchemeQuery(query2).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } +#endif + + Y_UNIT_TEST(CreateDropColumnTableNegative) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/ColumnTableTest"; + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `)" << tableName << R"(` ( Key Uint64, Value1 String, PRIMARY KEY (Key) ) WITH ( STORE = COLUMN, - PARTITION_BY_HASH_FUNCTION = "some_function" + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 0 );)"; auto result = session.ExecuteSchemeQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); diff --git a/ydb/library/yql/providers/common/provider/yql_provider.cpp b/ydb/library/yql/providers/common/provider/yql_provider.cpp index 494caf01108..3ccffdfd50d 100644 --- a/ydb/library/yql/providers/common/provider/yql_provider.cpp +++ b/ydb/library/yql/providers/common/provider/yql_provider.cpp @@ -204,6 +204,7 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { TMaybeNode<TExprList> columnFamilies; TVector<TCoNameValueTuple> tableSettings; TVector<TCoNameValueTuple> alterActions; + TMaybeNode<TCoAtom> tableType; for (auto child : node) { if (auto maybeTuple = child.Maybe<TCoNameValueTuple>()) { auto tuple = maybeTuple.Cast(); @@ -278,6 +279,9 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { for (const auto& item : tuple.Value().Cast<TCoNameValueTupleList>()) { alterActions.push_back(item); } + } else if (name == "tableType") { + YQL_ENSURE(tuple.Value().Maybe<TCoAtom>()); + tableType = tuple.Value().Cast<TCoAtom>(); } else { other.push_back(tuple); } @@ -321,6 +325,7 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { ret.ColumnFamilies = columnFamilies; ret.TableSettings = tableProfileSettings; ret.AlterActions = alterTableActions; + ret.TableType = tableType; return ret; } diff --git a/ydb/library/yql/providers/common/provider/yql_provider.h b/ydb/library/yql/providers/common/provider/yql_provider.h index 8d7d700f1bc..367ed0f9792 100644 --- a/ydb/library/yql/providers/common/provider/yql_provider.h +++ b/ydb/library/yql/providers/common/provider/yql_provider.h @@ -46,6 +46,7 @@ struct TWriteTableSettings { NNodes::TMaybeNode<NNodes::TExprList> ColumnFamilies; NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> TableSettings; NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> AlterActions; + NNodes::TMaybeNode<NNodes::TCoAtom> TableType; TWriteTableSettings(const NNodes::TCoNameValueTupleList& other) : Other(other) {} diff --git a/ydb/library/yql/sql/v1/node.h b/ydb/library/yql/sql/v1/node.h index 621eff8e814..ff59e682d2b 100644 --- a/ydb/library/yql/sql/v1/node.h +++ b/ydb/library/yql/sql/v1/node.h @@ -73,6 +73,11 @@ namespace NSQLTranslationV1 { DropColumn }; + enum class ETableType { + Table, + TableStore + }; + class TContext; class ITableKeys; class ISource; @@ -1169,6 +1174,7 @@ namespace NSQLTranslationV1 { TVector<TFamilyEntry> ColumnFamilies; TVector<TChangefeedDescription> Changefeeds; TTableSettings TableSettings; + ETableType TableType = ETableType::Table; }; struct TAlterTableParameters { @@ -1185,6 +1191,7 @@ namespace NSQLTranslationV1 { TVector<TChangefeedDescription> AlterChangefeeds; TVector<TIdentifier> DropChangefeeds; TMaybe<std::pair<TIdentifier, TIdentifier>> RenameIndexTo; + ETableType TableType = ETableType::Table; bool IsEmpty() const { return AddColumns.empty() && DropColumns.empty() && AlterColumns.empty() @@ -1363,7 +1370,7 @@ namespace NSQLTranslationV1 { TNodePtr BuildInputTables(TPosition pos, const TTableList& tables, bool inSubquery, TScopedStatePtr scoped); TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, const TCreateTableParameters& params, TScopedStatePtr scoped); TNodePtr BuildAlterTable(TPosition pos, const TTableRef& tr, const TAlterTableParameters& params, TScopedStatePtr scoped); - TNodePtr BuildDropTable(TPosition pos, const TTableRef& table, TScopedStatePtr scoped); + TNodePtr BuildDropTable(TPosition pos, const TTableRef& table, bool isTabletore, TScopedStatePtr scoped); TNodePtr BuildCreateUser(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TMaybe<TRoleParameters>& params, TScopedStatePtr scoped); TNodePtr BuildCreateGroup(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, TScopedStatePtr scoped); TNodePtr BuildAlterUser(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TRoleParameters& params, TScopedStatePtr scoped); diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp index 3b432d884eb..e16e543e7dd 100644 --- a/ydb/library/yql/sql/v1/query.cpp +++ b/ydb/library/yql/sql/v1/query.cpp @@ -873,6 +873,11 @@ public: opts = L(opts, Q(Y(Q("tableSettings"), Q(settings)))); } + + if (Params.TableType == ETableType::TableStore) { + opts = L(opts, Q(Y(Q("tableType"), Q("tableStore")))); + } + Add("block", Q(Y( Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Table.Service), Scoped->WrapCluster(Table.Cluster, ctx))), Y("let", "world", Y(TString(WriteName), "world", "sink", keys, Y("Void"), Q(opts))), @@ -1085,6 +1090,10 @@ public: opts = L(opts, Q(Y(Q("mode"), Q("alter")))); opts = L(opts, Q(Y(Q("actions"), Q(actions)))); + if (Params.TableType == ETableType::TableStore) { + opts = L(opts, Q(Y(Q("tableType"), Q("tableStore")))); + } + Add("block", Q(Y( Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Table.Service), Scoped->WrapCluster(Table.Cluster, ctx))), Y("let", "world", Y(TString(WriteName), "world", "sink", keys, Y("Void"), Q(opts))), @@ -1109,9 +1118,10 @@ TNodePtr BuildAlterTable(TPosition pos, const TTableRef& tr, const TAlterTablePa class TDropTableNode final: public TAstListNode { public: - TDropTableNode(TPosition pos, const TTableRef& tr, TScopedStatePtr scoped) + TDropTableNode(TPosition pos, const TTableRef& tr, bool isTabletore, TScopedStatePtr scoped) : TAstListNode(pos) , Table(tr) + , TableType(isTabletore ? ETableType::TableStore : ETableType::Table) , Scoped(scoped) { FakeSource = BuildFakeSource(pos); @@ -1125,9 +1135,17 @@ public: return false; } + auto opts = Y(); + + opts = L(opts, Q(Y(Q("mode"), Q("drop")))); + + if (TableType == ETableType::TableStore) { + opts = L(opts, Q(Y(Q("tableType"), Q("tableStore")))); + } + Add("block", Q(Y( Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Table.Service), Scoped->WrapCluster(Table.Cluster, ctx))), - Y("let", "world", Y(TString(WriteName), "world", "sink", keys, Y("Void"), Q(Y(Q(Y(Q("mode"), Q("drop"))))))), + Y("let", "world", Y(TString(WriteName), "world", "sink", keys, Y("Void"), Q(opts))), Y("return", ctx.PragmaAutoCommit ? Y(TString(CommitName), "world", "sink") : AstNode("world")) ))); @@ -1139,12 +1157,13 @@ public: } private: TTableRef Table; + ETableType TableType; TScopedStatePtr Scoped; TSourcePtr FakeSource; }; -TNodePtr BuildDropTable(TPosition pos, const TTableRef& tr, TScopedStatePtr scoped) { - return new TDropTableNode(pos, tr, scoped); +TNodePtr BuildDropTable(TPosition pos, const TTableRef& tr, bool isTabletore, TScopedStatePtr scoped) { + return new TDropTableNode(pos, tr, isTabletore, scoped); } class TCreateRole final: public TAstListNode { diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp index 2523a379b06..e4d0ea0ea3f 100644 --- a/ydb/library/yql/sql/v1/sql.cpp +++ b/ydb/library/yql/sql/v1/sql.cpp @@ -8603,16 +8603,16 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& Ctx.BodyPart(); const auto& rule = core.GetAlt_sql_stmt_core4().GetRule_create_table_stmt1(); const bool isTablestore = rule.GetToken2().GetId() == SQLv1LexerTokens::TOKEN_TABLESTORE; - if (isTablestore) { - Context().Error(GetPos(rule.GetToken2())) << "CREATE TABLESTORE is not supported yet"; - return false; - } + TTableRef tr; if (!SimpleTableRefImpl(rule.GetRule_simple_table_ref3(), tr)) { return false; } TCreateTableParameters params; + if (isTablestore) { + params.TableType = ETableType::TableStore; + } if (!CreateTableEntry(rule.GetRule_create_table_entry5(), params)) { return false; @@ -8624,12 +8624,14 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& } if (rule.HasBlock8()) { - Context().Error(GetPos(rule.GetBlock8().GetRule_table_inherits1().GetToken1())) << "INHERITS clause is not supported yet"; + Context().Error(GetPos(rule.GetBlock8().GetRule_table_inherits1().GetToken1())) + << "INHERITS clause is not supported yet"; return false; } - if (rule.HasBlock9()) { - Context().Error(GetPos(rule.GetBlock9().GetRule_table_partition_by1().GetToken1())) << "PARTITION BY clause is not supported yet"; + if (rule.HasBlock9() && isTablestore) { + Context().Error(GetPos(rule.GetBlock9().GetRule_table_partition_by1().GetToken1())) + << "PARTITION BY is not supported for TABLESTORE"; return false; } @@ -8640,7 +8642,8 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& } if (rule.HasBlock11()) { - Context().Error(GetPos(rule.GetBlock11().GetRule_table_tablestore1().GetToken1())) << "TABLESTORE clause is not supported yet"; + Context().Error(GetPos(rule.GetBlock11().GetRule_table_tablestore1().GetToken1())) + << "TABLESTORE clause is not supported yet"; return false; } @@ -8651,10 +8654,6 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& Ctx.BodyPart(); const auto& rule = core.GetAlt_sql_stmt_core5().GetRule_drop_table_stmt1(); const bool isTablestore = rule.GetToken2().GetId() == SQLv1LexerTokens::TOKEN_TABLESTORE; - if (isTablestore) { - Context().Error(GetPos(rule.GetToken2())) << "DROP TABLESTORE is not supported yet"; - return false; - } if (rule.HasBlock3()) { Context().Error(GetPos(rule.GetToken1())) << "IF EXISTS in " << humanStatementName << " is not supported."; @@ -8665,7 +8664,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& return false; } - AddStatementToBlocks(blocks, BuildDropTable(Ctx.Pos(), tr, Ctx.Scoped)); + AddStatementToBlocks(blocks, BuildDropTable(Ctx.Pos(), tr, isTablestore, Ctx.Scoped)); break; } case TRule_sql_stmt_core::kAltSqlStmtCore6: { @@ -8737,12 +8736,16 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& case TRule_sql_stmt_core::kAltSqlStmtCore15: { Ctx.BodyPart(); const auto& rule = core.GetAlt_sql_stmt_core15().GetRule_alter_table_stmt1(); + const bool isTablestore = rule.GetToken2().GetId() == SQLv1LexerTokens::TOKEN_TABLESTORE; TTableRef tr; if (!SimpleTableRefImpl(rule.GetRule_simple_table_ref3(), tr)) { return false; } TAlterTableParameters params; + if (isTablestore) { + params.TableType = ETableType::TableStore; + } if (!AlterTableAction(rule.GetRule_alter_table_action4(), params)) { return false; } |