diff options
author | Alexey Uzhegov <uzhegov37@yahoo.com> | 2024-01-30 14:32:09 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-01-30 13:32:09 +0200 |
commit | 479ab979a0980b72bcfcc6104f2297ca6fe2b795 (patch) | |
tree | 620abb07b7339c1d48e583bfe5d8de2a4a4d94e2 | |
parent | 2d0ace63155f45099719fcbb4dbd719bab36251f (diff) | |
download | ydb-479ab979a0980b72bcfcc6104f2297ca6fe2b795.tar.gz |
[YQ-1997] OR REPLACE grammar support for EXTERNAL DATA SOURCE and EXTERNAL TABLE (#1318)
21 files changed, 218 insertions, 76 deletions
diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp index f13d1155d7..ad65bde5da 100644 --- a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp +++ b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp @@ -37,6 +37,7 @@ void FillCreateExternalDataSourceDesc(NKikimrSchemeOp::TExternalDataSourceDescri externaDataSourceDesc.SetSourceType(GetOrEmpty(settings, "source_type")); externaDataSourceDesc.SetLocation(GetOrEmpty(settings, "location")); externaDataSourceDesc.SetInstallation(GetOrEmpty(settings, "installation")); + externaDataSourceDesc.SetReplaceIfExists(settings.GetReplaceIfExists()); TString authMethod = GetOrEmpty(settings, "auth_method"); if (authMethod == "NONE") { diff --git a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp index 1b68a5cf1a..72b65ebeac 100644 --- a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp +++ b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp @@ -850,10 +850,11 @@ public: return profilesPromise.GetFuture(); } - TFuture<TGenericResult> CreateTable(NYql::TKikimrTableMetadataPtr metadata, bool createDir, bool existingOk) override { + TFuture<TGenericResult> CreateTable(NYql::TKikimrTableMetadataPtr metadata, bool createDir, bool existingOk, bool replaceIfExists) override { Y_UNUSED(metadata); Y_UNUSED(createDir); Y_UNUSED(existingOk); + Y_UNUSED(replaceIfExists); return NotImplemented<TGenericResult>(); } @@ -1182,7 +1183,7 @@ public: TFuture<TGenericResult> CreateExternalTable(const TString& cluster, const NYql::TCreateExternalTableSettings& settings, - bool createDir, bool existingOk) override { + bool createDir, bool existingOk, bool replaceIfExists) override { using TRequest = TEvTxUserProxy::TEvProposeTransaction; try { @@ -1209,7 +1210,7 @@ public: schemeTx.SetFailedOnAlreadyExists(!existingOk); NKikimrSchemeOp::TExternalTableDescription& externalTableDesc = *schemeTx.MutableCreateExternalTable(); - NSchemeHelpers::FillCreateExternalTableColumnDesc(externalTableDesc, pathPair.second, settings); + NSchemeHelpers::FillCreateExternalTableColumnDesc(externalTableDesc, pathPair.second, replaceIfExists, settings); return SendSchemeRequest(ev.Release(), true); } catch (yexception& e) { diff --git a/ydb/core/kqp/gateway/utils/scheme_helpers.cpp b/ydb/core/kqp/gateway/utils/scheme_helpers.cpp index ccae2544e9..8b9657b260 100644 --- a/ydb/core/kqp/gateway/utils/scheme_helpers.cpp +++ b/ydb/core/kqp/gateway/utils/scheme_helpers.cpp @@ -61,12 +61,14 @@ bool SetDatabaseForLoginOperation(TString& result, bool getDomainLoginOnly, TMay void FillCreateExternalTableColumnDesc(NKikimrSchemeOp::TExternalTableDescription& externalTableDesc, const TString& name, + bool replaceIfExists, const NYql::TCreateExternalTableSettings& settings) { externalTableDesc.SetName(name); externalTableDesc.SetDataSourcePath(settings.DataSourcePath); externalTableDesc.SetLocation(settings.Location); externalTableDesc.SetSourceType("General"); + externalTableDesc.SetReplaceIfExists(replaceIfExists); Y_ENSURE(settings.ColumnOrder.size() == settings.Columns.size()); for (const auto& name : settings.ColumnOrder) { diff --git a/ydb/core/kqp/gateway/utils/scheme_helpers.h b/ydb/core/kqp/gateway/utils/scheme_helpers.h index 276ed3d641..21fe875287 100644 --- a/ydb/core/kqp/gateway/utils/scheme_helpers.h +++ b/ydb/core/kqp/gateway/utils/scheme_helpers.h @@ -29,6 +29,7 @@ bool SetDatabaseForLoginOperation(TString& result, bool getDomainLoginOnly, TMay void FillCreateExternalTableColumnDesc(NKikimrSchemeOp::TExternalTableDescription& externalTableDesc, const TString& name, + bool replaceIfExists, const NYql::TCreateExternalTableSettings& settings); std::pair<TString, TString> SplitPathByDirAndBaseNames(const TString& path); diff --git a/ydb/core/kqp/host/kqp_gateway_proxy.cpp b/ydb/core/kqp/host/kqp_gateway_proxy.cpp index f3b2201ffa..c153f18b27 100644 --- a/ydb/core/kqp/host/kqp_gateway_proxy.cpp +++ b/ydb/core/kqp/host/kqp_gateway_proxy.cpp @@ -444,7 +444,8 @@ public: return Gateway->LoadTableMetadata(cluster, table, settings); } - TFuture<TGenericResult> CreateTable(TKikimrTableMetadataPtr metadata, bool createDir, bool existingOk) override { + TFuture<TGenericResult> CreateTable(TKikimrTableMetadataPtr metadata, bool createDir, bool existingOk, bool replaceIfExists) override { + Y_UNUSED(replaceIfExists); CHECK_PREPARED_DDL(CreateTable); std::pair<TString, TString> pathPair; @@ -1243,7 +1244,7 @@ public: } TFuture<TGenericResult> CreateExternalTable(const TString& cluster, const TCreateExternalTableSettings& settings, - bool createDir, bool existingOk) override + bool createDir, bool existingOk, bool replaceIfExists) override { CHECK_PREPARED_DDL(CreateExternalTable); @@ -1270,13 +1271,13 @@ public: schemeTx.SetFailedOnAlreadyExists(!existingOk); NKikimrSchemeOp::TExternalTableDescription& externalTableDesc = *schemeTx.MutableCreateExternalTable(); - NSchemeHelpers::FillCreateExternalTableColumnDesc(externalTableDesc, pathPair.second, settings); + NSchemeHelpers::FillCreateExternalTableColumnDesc(externalTableDesc, pathPair.second, replaceIfExists, settings); TGenericResult result; result.SetSuccess(); phyTxRemover.Forget(); return MakeFuture(result); } else { - return Gateway->CreateExternalTable(cluster, settings, createDir, existingOk); + return Gateway->CreateExternalTable(cluster, settings, createDir, existingOk, replaceIfExists); } } diff --git a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp index e2df475c14..9c6243f35c 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp @@ -261,7 +261,7 @@ private: ? GetTableTypeFromString(settings.TableType.Cast()) : ETableType::Table; // v0 support - if (mode == "create" || mode == "create_if_not_exists") { + if (mode == "create" || mode == "create_if_not_exists" || mode == "create_or_replace") { if (!settings.Columns) { ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "No columns provided for create mode.")); @@ -762,7 +762,7 @@ public: ? settings.TableType.Cast() : Build<TCoAtom>(ctx, node->Pos()).Value("table").Done(); // v0 support auto mode = settings.Mode.Cast(); - if (mode == "create" || mode == "create_if_not_exists") { + if (mode == "create" || mode == "create_if_not_exists" || mode == "create_or_replace") { YQL_ENSURE(settings.Columns); YQL_ENSURE(!settings.Columns.Cast().Empty()); @@ -784,6 +784,7 @@ public: ? settings.Temporary.Cast() : Build<TCoAtom>(ctx, node->Pos()).Value("false").Done(); + auto replaceIfExists = (settings.Mode.Cast().Value() == "create_or_replace"); auto existringOk = (settings.Mode.Cast().Value() == "create_if_not_exists"); return Build<TKiCreateTable>(ctx, node->Pos()) @@ -800,9 +801,12 @@ public: .ColumnFamilies(settings.ColumnFamilies.Cast()) .TableSettings(settings.TableSettings.Cast()) .TableType(tableType) + .ReplaceIfExists<TCoAtom>() + .Value(replaceIfExists) + .Build() .ExistingOk<TCoAtom>() .Value(existringOk) - .Build() + .Build() .Done() .Ptr(); } else if (mode == "alter") { @@ -891,16 +895,19 @@ public: .Features(settings.Features) .Done() .Ptr(); - } else if (mode == "createObject" || mode == "createObjectIfNotExists") { + } else if (mode == "createObject" || mode == "createObjectIfNotExists" || mode == "createObjectOrReplace") { return Build<TKiCreateObject>(ctx, node->Pos()) .World(node->Child(0)) .DataSink(node->Child(1)) .ObjectId().Build(key.GetObjectId()) .TypeId().Build(key.GetObjectType()) .Features(settings.Features) + .ReplaceIfExists<TCoAtom>() + .Value(mode == "createObjectOrReplace") + .Build() .ExistingOk<TCoAtom>() .Value(mode == "createObjectIfNotExists") - .Build() + .Build() .Done() .Ptr(); } else if (mode == "alterObject") { diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp index 50c1de4dde..6940dc1816 100644 --- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp @@ -934,10 +934,11 @@ public: NThreading::TFuture<IKikimrGateway::TGenericResult> future; bool isColumn = (table.Metadata->StoreType == EStoreType::Column); bool existingOk = (maybeCreate.ExistingOk().Cast().Value() == "1"); + bool replaceIfExists = (maybeCreate.ReplaceIfExists().Cast().Value() == "1"); switch (tableTypeItem) { case ETableType::ExternalTable: { future = Gateway->CreateExternalTable(cluster, - ParseCreateExternalTableSettings(maybeCreate.Cast(), table.Metadata->TableSettings), true, existingOk); + ParseCreateExternalTableSettings(maybeCreate.Cast(), table.Metadata->TableSettings), true, existingOk, replaceIfExists); break; } case ETableType::TableStore: { diff --git a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json index aaccacb1a6..0a0185d689 100644 --- a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json +++ b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json @@ -131,7 +131,8 @@ {"Index": 10, "Name": "Changefeeds", "Type": "TCoChangefeedList"}, {"Index": 11, "Name": "TableType", "Type": "TCoAtom"}, {"Index": 12, "Name": "Temporary", "Type": "TCoAtom"}, - {"Index": 13, "Name": "ExistingOk", "Type": "TCoAtom"} + {"Index": 13, "Name": "ExistingOk", "Type": "TCoAtom"}, + {"Index": 14, "Name": "ReplaceIfExists", "Type": "TCoAtom"} ] }, { @@ -254,7 +255,8 @@ {"Index": 2, "Name": "ObjectId", "Type": "TCoAtom"}, {"Index": 3, "Name": "TypeId", "Type": "TCoAtom"}, {"Index": 4, "Name": "Features", "Type": "TCoNameValueTupleList"}, - {"Index": 5, "Name": "ExistingOk", "Type": "TCoAtom"} + {"Index": 5, "Name": "ExistingOk", "Type": "TCoAtom"}, + {"Index": 6, "Name": "ReplaceIfExists", "Type": "TCoAtom"} ] }, { diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index 99e3cdbd6b..8d79563400 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -790,7 +790,7 @@ public: virtual NThreading::TFuture<TTableMetadataResult> LoadTableMetadata( const TString& cluster, const TString& table, TLoadTableMetadataSettings settings) = 0; - virtual NThreading::TFuture<TGenericResult> CreateTable(TKikimrTableMetadataPtr metadata, bool createDir, bool existingOk = false) = 0; + virtual NThreading::TFuture<TGenericResult> CreateTable(TKikimrTableMetadataPtr metadata, bool createDir, bool existingOk = false, bool replaceIfExists = false) = 0; virtual NThreading::TFuture<TGenericResult> SendSchemeExecuterRequest(const TString& cluster, const TMaybe<TString>& requestType, @@ -843,7 +843,7 @@ public: virtual NThreading::TFuture<TGenericResult> DropTableStore(const TString& cluster, const TDropTableStoreSettings& settings) = 0; - virtual NThreading::TFuture<TGenericResult> CreateExternalTable(const TString& cluster, const TCreateExternalTableSettings& settings, bool createDir, bool existingOk) = 0; + virtual NThreading::TFuture<TGenericResult> CreateExternalTable(const TString& cluster, const TCreateExternalTableSettings& settings, bool createDir, bool existingOk, bool replaceIfExists) = 0; virtual NThreading::TFuture<TGenericResult> AlterExternalTable(const TString& cluster, const TAlterExternalTableSettings& settings) = 0; diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp index a072b09923..79d640f264 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp @@ -170,7 +170,7 @@ void TestCreateExternalTable(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGa settings.Columns.insert(std::make_pair("Column2", TKikimrColumnMetadata{"Column2", 0, "String", false})); settings.ColumnOrder.push_back("Column2"); - auto responseFuture = gateway->CreateExternalTable(TestCluster, settings, true, false); + auto responseFuture = gateway->CreateExternalTable(TestCluster, settings, true, false, false); responseFuture.Wait(); auto response = responseFuture.GetValue(); response.Issues().PrintTo(Cerr); diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 83e8533ba5..361006fcc0 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -1806,6 +1806,7 @@ message TExternalTableDescription { optional string Location = 6; repeated TColumnDescription Columns = 7; optional bytes Content = 8; + optional bool ReplaceIfExists = 9; // Only applicable for `create external table` operation } // Access without authorization @@ -1866,6 +1867,7 @@ message TExternalDataSourceDescription { optional string Installation = 6; optional TAuth Auth = 7; optional TExternalDataSourceProperties Properties = 8; + optional bool ReplaceIfExists = 9; } message TViewDescription { diff --git a/ydb/library/yql/sql/v1/SQLv1.g.in b/ydb/library/yql/sql/v1/SQLv1.g.in index 8fb43bb215..9be5921ef0 100644 --- a/ydb/library/yql/sql/v1/SQLv1.g.in +++ b/ydb/library/yql/sql/v1/SQLv1.g.in @@ -570,7 +570,7 @@ values_source_row: LPAREN expr_list RPAREN; simple_values_source: expr_list | select_stmt; -create_external_data_source_stmt: CREATE EXTERNAL DATA SOURCE (IF NOT EXISTS)? object_ref +create_external_data_source_stmt: CREATE (OR REPLACE)? EXTERNAL DATA SOURCE (IF NOT EXISTS)? object_ref with_table_settings ; @@ -623,7 +623,7 @@ object_features: object_feature | LPAREN object_feature (COMMA object_feature)* object_type_ref: an_id_or_type; -create_table_stmt: CREATE (TABLE | TABLESTORE | EXTERNAL TABLE) (IF NOT EXISTS)? simple_table_ref LPAREN create_table_entry (COMMA create_table_entry)* COMMA? RPAREN +create_table_stmt: CREATE (OR REPLACE)? (TABLE | TABLESTORE | EXTERNAL TABLE) (IF NOT EXISTS)? simple_table_ref LPAREN create_table_entry (COMMA create_table_entry)* COMMA? RPAREN table_inherits? table_partition_by? with_table_settings? diff --git a/ydb/library/yql/sql/v1/format/sql_format.cpp b/ydb/library/yql/sql/v1/format/sql_format.cpp index f6a47b4485..1e858be6b0 100644 --- a/ydb/library/yql/sql/v1/format/sql_format.cpp +++ b/ydb/library/yql/sql/v1/format/sql_format.cpp @@ -891,27 +891,24 @@ private: Visit(msg.GetToken1()); Visit(msg.GetBlock2()); Visit(msg.GetBlock3()); - Visit(msg.GetRule_simple_table_ref4()); - Visit(msg.GetToken5()); + Visit(msg.GetBlock4()); + Visit(msg.GetRule_simple_table_ref5()); + Visit(msg.GetToken6()); PushCurrentIndent(); NewLine(); - Visit(msg.GetRule_create_table_entry6()); - for (const auto& b : msg.GetBlock7()) { + Visit(msg.GetRule_create_table_entry7()); + for (const auto& b : msg.GetBlock8()) { Visit(b.GetToken1()); NewLine(); Visit(b.GetRule_create_table_entry2()); } - if (msg.HasBlock8()) { - Visit(msg.GetBlock8()); + if (msg.HasBlock9()) { + Visit(msg.GetBlock9()); } PopCurrentIndent(); NewLine(); - Visit(msg.GetToken9()); - if (msg.HasBlock10()) { - NewLine(); - Visit(msg.GetBlock10()); - } + Visit(msg.GetToken10()); if (msg.HasBlock11()) { NewLine(); Visit(msg.GetBlock11()); @@ -924,6 +921,10 @@ private: NewLine(); Visit(msg.GetBlock13()); } + if (msg.HasBlock14()) { + NewLine(); + Visit(msg.GetBlock14()); + } } void VisitDropTable(const TRule_drop_table_stmt& msg) { diff --git a/ydb/library/yql/sql/v1/format/sql_format_ut.cpp b/ydb/library/yql/sql/v1/format/sql_format_ut.cpp index b5bfca7ed0..cef8e235cc 100644 --- a/ydb/library/yql/sql/v1/format/sql_format_ut.cpp +++ b/ydb/library/yql/sql/v1/format/sql_format_ut.cpp @@ -354,6 +354,8 @@ Y_UNIT_TEST_SUITE(CheckSqlFormatter) { "CREATE EXTERNAL DATA SOURCE usEr WITH (a = \"b\");\n"}, {"creAte exTernAl daTa SouRce if not exists usEr With (a = \"b\")", "CREATE EXTERNAL DATA SOURCE IF NOT EXISTS usEr WITH (a = \"b\");\n"}, + {"creAte oR rePlaCe exTernAl daTa SouRce usEr With (a = \"b\")", + "CREATE OR REPLACE EXTERNAL DATA SOURCE usEr WITH (a = \"b\");\n"}, {"create external data source eds with (a=\"a\",b=\"b\",c = true)", "CREATE EXTERNAL DATA SOURCE eds WITH (\n\ta = \"a\",\n\tb = \"b\",\n\tc = TRUE\n);\n"}, {"alter external data source eds set a true, reset (b, c), set (x=y, z=false)", @@ -388,6 +390,8 @@ Y_UNIT_TEST_SUITE(CheckSqlFormatter) { TCases cases = { {"creAte exTernAl TabLe usEr (a int) With (a = \"b\")", "CREATE EXTERNAL TABLE usEr (\n\ta int\n)\nWITH (a = \"b\");\n"}, + {"creAte oR rePlaCe exTernAl TabLe usEr (a int) With (a = \"b\")", + "CREATE OR REPLACE EXTERNAL TABLE usEr (\n\ta int\n)\nWITH (a = \"b\");\n"}, {"creAte exTernAl TabLe iF NOt Exists usEr (a int) With (a = \"b\")", "CREATE EXTERNAL TABLE IF NOT EXISTS usEr (\n\ta int\n)\nWITH (a = \"b\");\n"}, {"create external table user (a int) with (a=\"b\",c=\"d\")", diff --git a/ydb/library/yql/sql/v1/node.h b/ydb/library/yql/sql/v1/node.h index 83c5d9972e..8bb0ba2030 100644 --- a/ydb/library/yql/sql/v1/node.h +++ b/ydb/library/yql/sql/v1/node.h @@ -1231,7 +1231,7 @@ namespace NSQLTranslationV1 { TNodePtr BuildUpsertObjectOperation(TPosition pos, const TString& objectId, const TString& typeId, std::map<TString, TDeferredAtom>&& features, const TObjectOperatorContext& context); TNodePtr BuildCreateObjectOperation(TPosition pos, const TString& objectId, const TString& typeId, - bool existingOk, std::map<TString, TDeferredAtom>&& features, const TObjectOperatorContext& context); + bool existingOk, bool replaceIfExists, std::map<TString, TDeferredAtom>&& features, const TObjectOperatorContext& context); TNodePtr BuildAlterObjectOperation(TPosition pos, const TString& secretId, const TString& typeId, std::map<TString, TDeferredAtom>&& features, std::set<TString>&& featuresToReset, const TObjectOperatorContext& context); TNodePtr BuildDropObjectOperation(TPosition pos, const TString& secretId, const TString& typeId, diff --git a/ydb/library/yql/sql/v1/object_processing.h b/ydb/library/yql/sql/v1/object_processing.h index e33ed5f84d..4114235ee6 100644 --- a/ydb/library/yql/sql/v1/object_processing.h +++ b/ydb/library/yql/sql/v1/object_processing.h @@ -40,19 +40,30 @@ private: std::set<TString> FeaturesToReset; protected: bool ExistingOk = false; + bool ReplaceIfExists = false; protected: virtual INode::TPtr BuildOptions() const override { - return Y(Q(Y(Q("mode"), Q(ExistingOk ? "createObjectIfNotExists" : "createObject")))); + TString mode; + if (ExistingOk) { + mode = "createObjectIfNotExists"; + } else if (ReplaceIfExists) { + mode = "createObjectOrReplace"; + } else { + mode = "createObject"; + } + + return Y(Q(Y(Q("mode"), Q(mode)))); } virtual INode::TPtr FillFeatures(INode::TPtr options) const override; public: TCreateObject(TPosition pos, const TString& objectId, - const TString& typeId, bool existingOk, std::map<TString, TDeferredAtom>&& features, std::set<TString>&& featuresToReset, const TObjectOperatorContext& context) + const TString& typeId, bool existingOk, bool replaceIfExists, std::map<TString, TDeferredAtom>&& features, std::set<TString>&& featuresToReset, const TObjectOperatorContext& context) : TBase(pos, objectId, typeId, context) , Features(std::move(features)) , FeaturesToReset(std::move(featuresToReset)) - , ExistingOk(existingOk) { - } + , ExistingOk(existingOk) + , ReplaceIfExists(replaceIfExists) { + } }; class TUpsertObject final: public TCreateObject { diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp index 7baae87545..4ea558ead1 100644 --- a/ydb/library/yql/sql/v1/query.cpp +++ b/ydb/library/yql/sql/v1/query.cpp @@ -808,11 +808,12 @@ TNodePtr BuildInputTables(TPosition pos, const TTableList& tables, bool inSubque class TCreateTableNode final: public TAstListNode { public: - TCreateTableNode(TPosition pos, const TTableRef& tr, bool existingOk, const TCreateTableParameters& params, TScopedStatePtr scoped) + TCreateTableNode(TPosition pos, const TTableRef& tr, bool existingOk, bool replaceIfExists, const TCreateTableParameters& params, TScopedStatePtr scoped) : TAstListNode(pos) , Table(tr) , Params(params) , ExistingOk(existingOk) + , ReplaceIfExists(replaceIfExists) , Scoped(scoped) { scoped->UseCluster(Table.Service, Table.Cluster); @@ -898,7 +899,13 @@ public: opts = Table.Options; } - opts = L(opts, Q(Y(Q("mode"), Q(ExistingOk ? "create_if_not_exists" : "create")))); + if (ExistingOk) { + opts = L(opts, Q(Y(Q("mode"), Q("create_if_not_exists")))); + } else if (ReplaceIfExists) { + opts = L(opts, Q(Y(Q("mode"), Q("create_or_replace")))); + } else { + opts = L(opts, Q(Y(Q("mode"), Q("create")))); + } THashSet<TString> columnFamilyNames; @@ -1149,12 +1156,13 @@ private: const TTableRef Table; const TCreateTableParameters Params; const bool ExistingOk; + const bool ReplaceIfExists; TScopedStatePtr Scoped; }; -TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, bool existingOk, const TCreateTableParameters& params, TScopedStatePtr scoped) +TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, bool existingOk, bool replaceIfExists, const TCreateTableParameters& params, TScopedStatePtr scoped) { - return new TCreateTableNode(pos, tr, existingOk, params, scoped); + return new TCreateTableNode(pos, tr, existingOk, replaceIfExists, params, scoped); } class TAlterTableNode final: public TAstListNode { @@ -2109,21 +2117,21 @@ private: TNodePtr BuildUpsertObjectOperation(TPosition pos, const TString& objectId, const TString& typeId, std::map<TString, TDeferredAtom>&& features, const TObjectOperatorContext& context) { - return new TUpsertObject(pos, objectId, typeId, false, std::move(features), std::set<TString>(), context); + return new TUpsertObject(pos, objectId, typeId, false, false, std::move(features), std::set<TString>(), context); } TNodePtr BuildCreateObjectOperation(TPosition pos, const TString& objectId, const TString& typeId, - bool existingOk, std::map<TString, TDeferredAtom>&& features, const TObjectOperatorContext& context) { - return new TCreateObject(pos, objectId, typeId, existingOk, std::move(features), std::set<TString>(), context); + bool existingOk, bool replaceIfExists, std::map<TString, TDeferredAtom>&& features, const TObjectOperatorContext& context) { + return new TCreateObject(pos, objectId, typeId, existingOk, replaceIfExists, std::move(features), std::set<TString>(), context); } TNodePtr BuildAlterObjectOperation(TPosition pos, const TString& secretId, const TString& typeId, std::map<TString, TDeferredAtom>&& features, std::set<TString>&& featuresToReset, const TObjectOperatorContext& context) { - return new TAlterObject(pos, secretId, typeId, false, std::move(features), std::move(featuresToReset), context); + return new TAlterObject(pos, secretId, typeId, false, false, std::move(features), std::move(featuresToReset), context); } TNodePtr BuildDropObjectOperation(TPosition pos, const TString& secretId, const TString& typeId, bool missingOk, std::map<TString, TDeferredAtom>&& options, const TObjectOperatorContext& context) { - return new TDropObject(pos, secretId, typeId, missingOk, std::move(options), std::set<TString>(), context); + return new TDropObject(pos, secretId, typeId, missingOk, false, std::move(options), std::set<TString>(), context); } TNodePtr BuildDropRoles(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TVector<TDeferredAtom>& toDrop, bool isUser, bool missingOk, TScopedStatePtr scoped) { diff --git a/ydb/library/yql/sql/v1/source.h b/ydb/library/yql/sql/v1/source.h index 4e2489d976..f1a91dec12 100644 --- a/ydb/library/yql/sql/v1/source.h +++ b/ydb/library/yql/sql/v1/source.h @@ -295,7 +295,7 @@ namespace NSQLTranslationV1 { TNodePtr BuildTopicKey(TPosition pos, const TDeferredAtom& cluster, const TDeferredAtom& name); TNodePtr BuildInputOptions(TPosition pos, const TTableHints& hints); TNodePtr BuildInputTables(TPosition pos, const TTableList& tables, bool inSubquery, TScopedStatePtr scoped); - TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, bool existingOk, const TCreateTableParameters& params, TScopedStatePtr scoped); + TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, bool existingOk, bool replaceIfExists, const TCreateTableParameters& params, TScopedStatePtr scoped); TNodePtr BuildAlterTable(TPosition pos, const TTableRef& tr, const TAlterTableParameters& params, TScopedStatePtr scoped); TNodePtr BuildDropTable(TPosition pos, const TTableRef& table, bool missingOk, ETableType tableType, TScopedStatePtr scoped); TNodePtr BuildWriteTable(TPosition pos, const TString& label, const TTableRef& table, EWriteColumnMode mode, TNodePtr options, diff --git a/ydb/library/yql/sql/v1/sql_query.cpp b/ydb/library/yql/sql/v1/sql_query.cpp index 931614d1dc..79f9c34696 100644 --- a/ydb/library/yql/sql/v1/sql_query.cpp +++ b/ydb/library/yql/sql/v1/sql_query.cpp @@ -154,7 +154,17 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& case TRule_sql_stmt_core::kAltSqlStmtCore4: { Ctx.BodyPart(); const auto& rule = core.GetAlt_sql_stmt_core4().GetRule_create_table_stmt1(); - const auto& block = rule.GetBlock2(); + + bool replaceIfExists = false; + if (rule.HasBlock2()) { // OR REPLACE + replaceIfExists = true; + Y_DEBUG_ABORT_UNLESS( + rule.GetBlock2().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_OR && + rule.GetBlock2().GetToken2().GetId() == SQLv1LexerTokens::TOKEN_REPLACE + ); + } + + const auto& block = rule.GetBlock3(); ETableType tableType = ETableType::Table; if (block.HasAlt2() && block.GetAlt2().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_TABLESTORE) { tableType = ETableType::TableStore; @@ -163,57 +173,63 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& } bool existingOk = false; - if (rule.HasBlock3()) { // IF NOT EXISTS + if (rule.HasBlock4()) { // IF NOT EXISTS existingOk = true; Y_DEBUG_ABORT_UNLESS( - rule.GetBlock3().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_IF && - rule.GetBlock3().GetToken2().GetId() == SQLv1LexerTokens::TOKEN_NOT && - rule.GetBlock3().GetToken3().GetId() == SQLv1LexerTokens::TOKEN_EXISTS + rule.GetBlock4().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_IF && + rule.GetBlock4().GetToken2().GetId() == SQLv1LexerTokens::TOKEN_NOT && + rule.GetBlock4().GetToken3().GetId() == SQLv1LexerTokens::TOKEN_EXISTS ); } + if (replaceIfExists && tableType != ETableType::ExternalTable) { + Context().Error(GetPos(rule.GetBlock2().GetToken1())) + << "OR REPLACE feature is supported only for EXTERNAL DATA SOURCE and EXTERNAL TABLE"; + return false; + } + TTableRef tr; - if (!SimpleTableRefImpl(rule.GetRule_simple_table_ref4(), tr)) { + if (!SimpleTableRefImpl(rule.GetRule_simple_table_ref5(), tr)) { return false; } TCreateTableParameters params{.TableType=tableType}; - if (!CreateTableEntry(rule.GetRule_create_table_entry6(), params)) { + if (!CreateTableEntry(rule.GetRule_create_table_entry7(), params)) { return false; } - for (auto& block: rule.GetBlock7()) { + for (auto& block: rule.GetBlock8()) { if (!CreateTableEntry(block.GetRule_create_table_entry2(), params)) { return false; } } - if (rule.HasBlock10()) { - Context().Error(GetPos(rule.GetBlock10().GetRule_table_inherits1().GetToken1())) + if (rule.HasBlock11()) { + Context().Error(GetPos(rule.GetBlock11().GetRule_table_inherits1().GetToken1())) << "INHERITS clause is not supported yet"; return false; } - if (rule.HasBlock11()) { + if (rule.HasBlock12()) { if (tableType == ETableType::TableStore) { - Context().Error(GetPos(rule.GetBlock11().GetRule_table_partition_by1().GetToken1())) + Context().Error(GetPos(rule.GetBlock12().GetRule_table_partition_by1().GetToken1())) << "PARTITION BY is not supported for TABLESTORE"; return false; } - const auto list = rule.GetBlock11().GetRule_table_partition_by1().GetRule_pure_column_list4(); + const auto list = rule.GetBlock12().GetRule_table_partition_by1().GetRule_pure_column_list4(); params.PartitionByColumns.push_back(IdEx(list.GetRule_an_id2(), *this)); for (auto& node : list.GetBlock3()) { params.PartitionByColumns.push_back(IdEx(node.GetRule_an_id2(), *this)); } } - if (rule.HasBlock12()) { - if (!CreateTableSettings(rule.GetBlock12().GetRule_with_table_settings1(), params)) { + if (rule.HasBlock13()) { + if (!CreateTableSettings(rule.GetBlock13().GetRule_with_table_settings1(), params)) { return false; } } - if (rule.HasBlock13()) { - Context().Error(GetPos(rule.GetBlock13().GetRule_table_tablestore1().GetToken1())) + if (rule.HasBlock14()) { + Context().Error(GetPos(rule.GetBlock14().GetRule_table_tablestore1().GetToken1())) << "TABLESTORE clause is not supported yet"; return false; } @@ -222,7 +238,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& return false; } - AddStatementToBlocks(blocks, BuildCreateTable(Ctx.Pos(), tr, existingOk, params, Ctx.Scoped)); + AddStatementToBlocks(blocks, BuildCreateTable(Ctx.Pos(), tr, existingOk, replaceIfExists, params, Ctx.Scoped)); break; } case TRule_sql_stmt_core::kAltSqlStmtCore5: { @@ -677,7 +693,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& } } - AddStatementToBlocks(blocks, BuildCreateObjectOperation(Ctx.Pos(), objectId, typeId, existingOk, std::move(kv), context)); + AddStatementToBlocks(blocks, BuildCreateObjectOperation(Ctx.Pos(), objectId, typeId, existingOk, false, std::move(kv), context)); break; } case TRule_sql_stmt_core::kAltSqlStmtCore28: { @@ -734,33 +750,42 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& break; } case TRule_sql_stmt_core::kAltSqlStmtCore30: { - // create_external_data_source_stmt: CREATE EXTERNAL DATA SOURCE (IF NOT EXISTS)? name WITH (k=v,...); + // create_external_data_source_stmt: CREATE (OR REPLACE)? EXTERNAL DATA SOURCE (IF NOT EXISTS)? name WITH (k=v,...); auto& node = core.GetAlt_sql_stmt_core30().GetRule_create_external_data_source_stmt1(); TObjectOperatorContext context(Ctx.Scoped); - if (node.GetRule_object_ref6().HasBlock1()) { - if (!ClusterExpr(node.GetRule_object_ref6().GetBlock1().GetRule_cluster_expr1(), + if (node.GetRule_object_ref7().HasBlock1()) { + if (!ClusterExpr(node.GetRule_object_ref7().GetBlock1().GetRule_cluster_expr1(), false, context.ServiceId, context.Cluster)) { return false; } } + bool replaceIfExists = false; + if (node.HasBlock2()) { // OR REPLACE + replaceIfExists = true; + Y_DEBUG_ABORT_UNLESS( + node.GetBlock2().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_OR && + node.GetBlock2().GetToken2().GetId() == SQLv1LexerTokens::TOKEN_REPLACE + ); + } + bool existingOk = false; - if (node.HasBlock5()) { // IF NOT EXISTS + if (node.HasBlock6()) { // IF NOT EXISTS existingOk = true; Y_DEBUG_ABORT_UNLESS( - node.GetBlock5().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_IF && - node.GetBlock5().GetToken2().GetId() == SQLv1LexerTokens::TOKEN_NOT && - node.GetBlock5().GetToken3().GetId() == SQLv1LexerTokens::TOKEN_EXISTS + node.GetBlock6().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_IF && + node.GetBlock6().GetToken2().GetId() == SQLv1LexerTokens::TOKEN_NOT && + node.GetBlock6().GetToken3().GetId() == SQLv1LexerTokens::TOKEN_EXISTS ); } - const TString& objectId = Id(node.GetRule_object_ref6().GetRule_id_or_at2(), *this).second; + const TString& objectId = Id(node.GetRule_object_ref7().GetRule_id_or_at2(), *this).second; std::map<TString, TDeferredAtom> kv; - if (!ParseExternalDataSourceSettings(kv, node.GetRule_with_table_settings7())) { + if (!ParseExternalDataSourceSettings(kv, node.GetRule_with_table_settings8())) { return false; } - AddStatementToBlocks(blocks, BuildCreateObjectOperation(Ctx.Pos(), BuildTablePath(Ctx.GetPrefixPath(context.ServiceId, context.Cluster), objectId), "EXTERNAL_DATA_SOURCE", existingOk, std::move(kv), context)); + AddStatementToBlocks(blocks, BuildCreateObjectOperation(Ctx.Pos(), BuildTablePath(Ctx.GetPrefixPath(context.ServiceId, context.Cluster), objectId), "EXTERNAL_DATA_SOURCE", existingOk, replaceIfExists, std::move(kv), context)); break; } case TRule_sql_stmt_core::kAltSqlStmtCore31: { @@ -1085,6 +1110,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& BuildTablePath(Ctx.GetPrefixPath(context.ServiceId, context.Cluster), objectId), TypeId, false, + false, std::move(features), context)); break; diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp index 178f2a7996..6da68c791b 100644 --- a/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/ydb/library/yql/sql/v1/sql_ut.cpp @@ -5582,6 +5582,50 @@ Y_UNIT_TEST_SUITE(ExternalDataSource) { UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); } + Y_UNIT_TEST(CreateExternalDataSourceOrReplace) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + CREATE OR REPLACE EXTERNAL DATA SOURCE MyDataSource WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="my-bucket", + AUTH_METHOD="NONE" + ); + )"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"NONE") '('"location" '"my-bucket") '('"source_type" '"ObjectStorage"))#"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObjectOrReplace")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateOrReplaceForUnsupportedTableTypesShouldFail) { + ExpectFailWithError(R"sql( + USE plato; + CREATE OR REPLACE TABLE t (a int32 not null, primary key(a, a)); + )sql" , "<main>:3:23: Error: OR REPLACE feature is supported only for EXTERNAL DATA SOURCE and EXTERNAL TABLE\n"); + + ExpectFailWithError(R"sql( + USE plato; + CREATE OR REPLACE TABLE t ( + Key Uint64, + Value1 String, + PRIMARY KEY (Key) + ) + WITH ( + STORE = COLUMN, + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10 + ); + )sql" , "<main>:3:23: Error: OR REPLACE feature is supported only for EXTERNAL DATA SOURCE and EXTERNAL TABLE\n"); + } + Y_UNIT_TEST(CreateExternalDataSourceWithBadArguments) { ExpectFailWithError(R"sql( USE plato; @@ -5913,6 +5957,31 @@ Y_UNIT_TEST_SUITE(ExternalTable) { UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); } + Y_UNIT_TEST(CreateExternalTableOrReplace) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + CREATE OR REPLACE EXTERNAL TABLE mytable ( + a int + ) WITH ( + DATA_SOURCE="/Root/mydatasource", + LOCATION="/folder1/*" + ); + )"); + UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('data_source_path (String '"/Root/mydatasource")) '('location (String '"/folder1/*")))) '('tableType 'externalTable)))))#"); + UNIT_ASSERT_STRING_CONTAINS(line, "create_or_replace"); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + Y_UNIT_TEST(AlterExternalTableAddColumn) { NYql::TAstParseResult res = SqlToYql(R"sql( USE plato; @@ -6230,7 +6299,7 @@ Y_UNIT_TEST_SUITE(TViewSyntaxTest) { ); UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "SECURITY_INVOKER option must be explicitly enabled"); } - + Y_UNIT_TEST(CreateViewFromTable) { constexpr const char* path = "/PathPrefix/TheView"; constexpr const char* query = R"( diff --git a/ydb/services/metadata/abstract/parsing.h b/ydb/services/metadata/abstract/parsing.h index 49c882a2ab..6e6743327b 100644 --- a/ydb/services/metadata/abstract/parsing.h +++ b/ydb/services/metadata/abstract/parsing.h @@ -13,6 +13,7 @@ namespace NYql { namespace NObjectOptionsParsing { Y_HAS_MEMBER(ExistingOk); // for create Y_HAS_MEMBER(MissingOk); // for drop +Y_HAS_MEMBER(ReplaceIfExists); // for create } // namespace NObjectOptionsParsing class TObjectSettingsImpl { @@ -24,6 +25,7 @@ private: YDB_READONLY_DEF(TString, ObjectId); YDB_READONLY_DEF(bool, ExistingOk); // for create YDB_READONLY_DEF(bool, MissingOk); // for drop + YDB_READONLY_DEF(bool, ReplaceIfExists); // for create TFeatures Features; std::shared_ptr<TFeaturesExtractor> FeaturesExtractor; public: @@ -45,6 +47,9 @@ public: bool DeserializeFromKi(const TKiObject& data) { ObjectId = data.ObjectId(); TypeId = data.TypeId(); + if constexpr (NObjectOptionsParsing::THasReplaceIfExists<TKiObject>::value) { + ReplaceIfExists = (data.ReplaceIfExists().Value() == "1"); + } if constexpr (NObjectOptionsParsing::THasExistingOk<TKiObject>::value) { ExistingOk = (data.ExistingOk().Value() == "1"); } |