aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexey Uzhegov <uzhegov37@yahoo.com>2024-01-30 14:32:09 +0300
committerGitHub <noreply@github.com>2024-01-30 13:32:09 +0200
commit479ab979a0980b72bcfcc6104f2297ca6fe2b795 (patch)
tree620abb07b7339c1d48e583bfe5d8de2a4a4d94e2
parent2d0ace63155f45099719fcbb4dbd719bab36251f (diff)
downloadydb-479ab979a0980b72bcfcc6104f2297ca6fe2b795.tar.gz
[YQ-1997] OR REPLACE grammar support for EXTERNAL DATA SOURCE and EXTERNAL TABLE (#1318)
-rw-r--r--ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp1
-rw-r--r--ydb/core/kqp/gateway/kqp_ic_gateway.cpp7
-rw-r--r--ydb/core/kqp/gateway/utils/scheme_helpers.cpp2
-rw-r--r--ydb/core/kqp/gateway/utils/scheme_helpers.h1
-rw-r--r--ydb/core/kqp/host/kqp_gateway_proxy.cpp9
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_datasink.cpp17
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_exec.cpp3
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_expr_nodes.json6
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_gateway.h4
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp2
-rw-r--r--ydb/core/protos/flat_scheme_op.proto2
-rw-r--r--ydb/library/yql/sql/v1/SQLv1.g.in4
-rw-r--r--ydb/library/yql/sql/v1/format/sql_format.cpp23
-rw-r--r--ydb/library/yql/sql/v1/format/sql_format_ut.cpp4
-rw-r--r--ydb/library/yql/sql/v1/node.h2
-rw-r--r--ydb/library/yql/sql/v1/object_processing.h19
-rw-r--r--ydb/library/yql/sql/v1/query.cpp26
-rw-r--r--ydb/library/yql/sql/v1/source.h2
-rw-r--r--ydb/library/yql/sql/v1/sql_query.cpp84
-rw-r--r--ydb/library/yql/sql/v1/sql_ut.cpp71
-rw-r--r--ydb/services/metadata/abstract/parsing.h5
21 files changed, 218 insertions, 76 deletions
diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp
index f13d1155d7..ad65bde5da 100644
--- a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp
+++ b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp
@@ -37,6 +37,7 @@ void FillCreateExternalDataSourceDesc(NKikimrSchemeOp::TExternalDataSourceDescri
externaDataSourceDesc.SetSourceType(GetOrEmpty(settings, "source_type"));
externaDataSourceDesc.SetLocation(GetOrEmpty(settings, "location"));
externaDataSourceDesc.SetInstallation(GetOrEmpty(settings, "installation"));
+ externaDataSourceDesc.SetReplaceIfExists(settings.GetReplaceIfExists());
TString authMethod = GetOrEmpty(settings, "auth_method");
if (authMethod == "NONE") {
diff --git a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp
index 1b68a5cf1a..72b65ebeac 100644
--- a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp
+++ b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp
@@ -850,10 +850,11 @@ public:
return profilesPromise.GetFuture();
}
- TFuture<TGenericResult> CreateTable(NYql::TKikimrTableMetadataPtr metadata, bool createDir, bool existingOk) override {
+ TFuture<TGenericResult> CreateTable(NYql::TKikimrTableMetadataPtr metadata, bool createDir, bool existingOk, bool replaceIfExists) override {
Y_UNUSED(metadata);
Y_UNUSED(createDir);
Y_UNUSED(existingOk);
+ Y_UNUSED(replaceIfExists);
return NotImplemented<TGenericResult>();
}
@@ -1182,7 +1183,7 @@ public:
TFuture<TGenericResult> CreateExternalTable(const TString& cluster,
const NYql::TCreateExternalTableSettings& settings,
- bool createDir, bool existingOk) override {
+ bool createDir, bool existingOk, bool replaceIfExists) override {
using TRequest = TEvTxUserProxy::TEvProposeTransaction;
try {
@@ -1209,7 +1210,7 @@ public:
schemeTx.SetFailedOnAlreadyExists(!existingOk);
NKikimrSchemeOp::TExternalTableDescription& externalTableDesc = *schemeTx.MutableCreateExternalTable();
- NSchemeHelpers::FillCreateExternalTableColumnDesc(externalTableDesc, pathPair.second, settings);
+ NSchemeHelpers::FillCreateExternalTableColumnDesc(externalTableDesc, pathPair.second, replaceIfExists, settings);
return SendSchemeRequest(ev.Release(), true);
}
catch (yexception& e) {
diff --git a/ydb/core/kqp/gateway/utils/scheme_helpers.cpp b/ydb/core/kqp/gateway/utils/scheme_helpers.cpp
index ccae2544e9..8b9657b260 100644
--- a/ydb/core/kqp/gateway/utils/scheme_helpers.cpp
+++ b/ydb/core/kqp/gateway/utils/scheme_helpers.cpp
@@ -61,12 +61,14 @@ bool SetDatabaseForLoginOperation(TString& result, bool getDomainLoginOnly, TMay
void FillCreateExternalTableColumnDesc(NKikimrSchemeOp::TExternalTableDescription& externalTableDesc,
const TString& name,
+ bool replaceIfExists,
const NYql::TCreateExternalTableSettings& settings)
{
externalTableDesc.SetName(name);
externalTableDesc.SetDataSourcePath(settings.DataSourcePath);
externalTableDesc.SetLocation(settings.Location);
externalTableDesc.SetSourceType("General");
+ externalTableDesc.SetReplaceIfExists(replaceIfExists);
Y_ENSURE(settings.ColumnOrder.size() == settings.Columns.size());
for (const auto& name : settings.ColumnOrder) {
diff --git a/ydb/core/kqp/gateway/utils/scheme_helpers.h b/ydb/core/kqp/gateway/utils/scheme_helpers.h
index 276ed3d641..21fe875287 100644
--- a/ydb/core/kqp/gateway/utils/scheme_helpers.h
+++ b/ydb/core/kqp/gateway/utils/scheme_helpers.h
@@ -29,6 +29,7 @@ bool SetDatabaseForLoginOperation(TString& result, bool getDomainLoginOnly, TMay
void FillCreateExternalTableColumnDesc(NKikimrSchemeOp::TExternalTableDescription& externalTableDesc,
const TString& name,
+ bool replaceIfExists,
const NYql::TCreateExternalTableSettings& settings);
std::pair<TString, TString> SplitPathByDirAndBaseNames(const TString& path);
diff --git a/ydb/core/kqp/host/kqp_gateway_proxy.cpp b/ydb/core/kqp/host/kqp_gateway_proxy.cpp
index f3b2201ffa..c153f18b27 100644
--- a/ydb/core/kqp/host/kqp_gateway_proxy.cpp
+++ b/ydb/core/kqp/host/kqp_gateway_proxy.cpp
@@ -444,7 +444,8 @@ public:
return Gateway->LoadTableMetadata(cluster, table, settings);
}
- TFuture<TGenericResult> CreateTable(TKikimrTableMetadataPtr metadata, bool createDir, bool existingOk) override {
+ TFuture<TGenericResult> CreateTable(TKikimrTableMetadataPtr metadata, bool createDir, bool existingOk, bool replaceIfExists) override {
+ Y_UNUSED(replaceIfExists);
CHECK_PREPARED_DDL(CreateTable);
std::pair<TString, TString> pathPair;
@@ -1243,7 +1244,7 @@ public:
}
TFuture<TGenericResult> CreateExternalTable(const TString& cluster, const TCreateExternalTableSettings& settings,
- bool createDir, bool existingOk) override
+ bool createDir, bool existingOk, bool replaceIfExists) override
{
CHECK_PREPARED_DDL(CreateExternalTable);
@@ -1270,13 +1271,13 @@ public:
schemeTx.SetFailedOnAlreadyExists(!existingOk);
NKikimrSchemeOp::TExternalTableDescription& externalTableDesc = *schemeTx.MutableCreateExternalTable();
- NSchemeHelpers::FillCreateExternalTableColumnDesc(externalTableDesc, pathPair.second, settings);
+ NSchemeHelpers::FillCreateExternalTableColumnDesc(externalTableDesc, pathPair.second, replaceIfExists, settings);
TGenericResult result;
result.SetSuccess();
phyTxRemover.Forget();
return MakeFuture(result);
} else {
- return Gateway->CreateExternalTable(cluster, settings, createDir, existingOk);
+ return Gateway->CreateExternalTable(cluster, settings, createDir, existingOk, replaceIfExists);
}
}
diff --git a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp
index e2df475c14..9c6243f35c 100644
--- a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp
@@ -261,7 +261,7 @@ private:
? GetTableTypeFromString(settings.TableType.Cast())
: ETableType::Table; // v0 support
- if (mode == "create" || mode == "create_if_not_exists") {
+ if (mode == "create" || mode == "create_if_not_exists" || mode == "create_or_replace") {
if (!settings.Columns) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder()
<< "No columns provided for create mode."));
@@ -762,7 +762,7 @@ public:
? settings.TableType.Cast()
: Build<TCoAtom>(ctx, node->Pos()).Value("table").Done(); // v0 support
auto mode = settings.Mode.Cast();
- if (mode == "create" || mode == "create_if_not_exists") {
+ if (mode == "create" || mode == "create_if_not_exists" || mode == "create_or_replace") {
YQL_ENSURE(settings.Columns);
YQL_ENSURE(!settings.Columns.Cast().Empty());
@@ -784,6 +784,7 @@ public:
? settings.Temporary.Cast()
: Build<TCoAtom>(ctx, node->Pos()).Value("false").Done();
+ auto replaceIfExists = (settings.Mode.Cast().Value() == "create_or_replace");
auto existringOk = (settings.Mode.Cast().Value() == "create_if_not_exists");
return Build<TKiCreateTable>(ctx, node->Pos())
@@ -800,9 +801,12 @@ public:
.ColumnFamilies(settings.ColumnFamilies.Cast())
.TableSettings(settings.TableSettings.Cast())
.TableType(tableType)
+ .ReplaceIfExists<TCoAtom>()
+ .Value(replaceIfExists)
+ .Build()
.ExistingOk<TCoAtom>()
.Value(existringOk)
- .Build()
+ .Build()
.Done()
.Ptr();
} else if (mode == "alter") {
@@ -891,16 +895,19 @@ public:
.Features(settings.Features)
.Done()
.Ptr();
- } else if (mode == "createObject" || mode == "createObjectIfNotExists") {
+ } else if (mode == "createObject" || mode == "createObjectIfNotExists" || mode == "createObjectOrReplace") {
return Build<TKiCreateObject>(ctx, node->Pos())
.World(node->Child(0))
.DataSink(node->Child(1))
.ObjectId().Build(key.GetObjectId())
.TypeId().Build(key.GetObjectType())
.Features(settings.Features)
+ .ReplaceIfExists<TCoAtom>()
+ .Value(mode == "createObjectOrReplace")
+ .Build()
.ExistingOk<TCoAtom>()
.Value(mode == "createObjectIfNotExists")
- .Build()
+ .Build()
.Done()
.Ptr();
} else if (mode == "alterObject") {
diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp
index 50c1de4dde..6940dc1816 100644
--- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp
@@ -934,10 +934,11 @@ public:
NThreading::TFuture<IKikimrGateway::TGenericResult> future;
bool isColumn = (table.Metadata->StoreType == EStoreType::Column);
bool existingOk = (maybeCreate.ExistingOk().Cast().Value() == "1");
+ bool replaceIfExists = (maybeCreate.ReplaceIfExists().Cast().Value() == "1");
switch (tableTypeItem) {
case ETableType::ExternalTable: {
future = Gateway->CreateExternalTable(cluster,
- ParseCreateExternalTableSettings(maybeCreate.Cast(), table.Metadata->TableSettings), true, existingOk);
+ ParseCreateExternalTableSettings(maybeCreate.Cast(), table.Metadata->TableSettings), true, existingOk, replaceIfExists);
break;
}
case ETableType::TableStore: {
diff --git a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json
index aaccacb1a6..0a0185d689 100644
--- a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json
+++ b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json
@@ -131,7 +131,8 @@
{"Index": 10, "Name": "Changefeeds", "Type": "TCoChangefeedList"},
{"Index": 11, "Name": "TableType", "Type": "TCoAtom"},
{"Index": 12, "Name": "Temporary", "Type": "TCoAtom"},
- {"Index": 13, "Name": "ExistingOk", "Type": "TCoAtom"}
+ {"Index": 13, "Name": "ExistingOk", "Type": "TCoAtom"},
+ {"Index": 14, "Name": "ReplaceIfExists", "Type": "TCoAtom"}
]
},
{
@@ -254,7 +255,8 @@
{"Index": 2, "Name": "ObjectId", "Type": "TCoAtom"},
{"Index": 3, "Name": "TypeId", "Type": "TCoAtom"},
{"Index": 4, "Name": "Features", "Type": "TCoNameValueTupleList"},
- {"Index": 5, "Name": "ExistingOk", "Type": "TCoAtom"}
+ {"Index": 5, "Name": "ExistingOk", "Type": "TCoAtom"},
+ {"Index": 6, "Name": "ReplaceIfExists", "Type": "TCoAtom"}
]
},
{
diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h
index 99e3cdbd6b..8d79563400 100644
--- a/ydb/core/kqp/provider/yql_kikimr_gateway.h
+++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h
@@ -790,7 +790,7 @@ public:
virtual NThreading::TFuture<TTableMetadataResult> LoadTableMetadata(
const TString& cluster, const TString& table, TLoadTableMetadataSettings settings) = 0;
- virtual NThreading::TFuture<TGenericResult> CreateTable(TKikimrTableMetadataPtr metadata, bool createDir, bool existingOk = false) = 0;
+ virtual NThreading::TFuture<TGenericResult> CreateTable(TKikimrTableMetadataPtr metadata, bool createDir, bool existingOk = false, bool replaceIfExists = false) = 0;
virtual NThreading::TFuture<TGenericResult> SendSchemeExecuterRequest(const TString& cluster,
const TMaybe<TString>& requestType,
@@ -843,7 +843,7 @@ public:
virtual NThreading::TFuture<TGenericResult> DropTableStore(const TString& cluster, const TDropTableStoreSettings& settings) = 0;
- virtual NThreading::TFuture<TGenericResult> CreateExternalTable(const TString& cluster, const TCreateExternalTableSettings& settings, bool createDir, bool existingOk) = 0;
+ virtual NThreading::TFuture<TGenericResult> CreateExternalTable(const TString& cluster, const TCreateExternalTableSettings& settings, bool createDir, bool existingOk, bool replaceIfExists) = 0;
virtual NThreading::TFuture<TGenericResult> AlterExternalTable(const TString& cluster, const TAlterExternalTableSettings& settings) = 0;
diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp
index a072b09923..79d640f264 100644
--- a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp
@@ -170,7 +170,7 @@ void TestCreateExternalTable(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGa
settings.Columns.insert(std::make_pair("Column2", TKikimrColumnMetadata{"Column2", 0, "String", false}));
settings.ColumnOrder.push_back("Column2");
- auto responseFuture = gateway->CreateExternalTable(TestCluster, settings, true, false);
+ auto responseFuture = gateway->CreateExternalTable(TestCluster, settings, true, false, false);
responseFuture.Wait();
auto response = responseFuture.GetValue();
response.Issues().PrintTo(Cerr);
diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto
index 83e8533ba5..361006fcc0 100644
--- a/ydb/core/protos/flat_scheme_op.proto
+++ b/ydb/core/protos/flat_scheme_op.proto
@@ -1806,6 +1806,7 @@ message TExternalTableDescription {
optional string Location = 6;
repeated TColumnDescription Columns = 7;
optional bytes Content = 8;
+ optional bool ReplaceIfExists = 9; // Only applicable for `create external table` operation
}
// Access without authorization
@@ -1866,6 +1867,7 @@ message TExternalDataSourceDescription {
optional string Installation = 6;
optional TAuth Auth = 7;
optional TExternalDataSourceProperties Properties = 8;
+ optional bool ReplaceIfExists = 9;
}
message TViewDescription {
diff --git a/ydb/library/yql/sql/v1/SQLv1.g.in b/ydb/library/yql/sql/v1/SQLv1.g.in
index 8fb43bb215..9be5921ef0 100644
--- a/ydb/library/yql/sql/v1/SQLv1.g.in
+++ b/ydb/library/yql/sql/v1/SQLv1.g.in
@@ -570,7 +570,7 @@ values_source_row: LPAREN expr_list RPAREN;
simple_values_source: expr_list | select_stmt;
-create_external_data_source_stmt: CREATE EXTERNAL DATA SOURCE (IF NOT EXISTS)? object_ref
+create_external_data_source_stmt: CREATE (OR REPLACE)? EXTERNAL DATA SOURCE (IF NOT EXISTS)? object_ref
with_table_settings
;
@@ -623,7 +623,7 @@ object_features: object_feature | LPAREN object_feature (COMMA object_feature)*
object_type_ref: an_id_or_type;
-create_table_stmt: CREATE (TABLE | TABLESTORE | EXTERNAL TABLE) (IF NOT EXISTS)? simple_table_ref LPAREN create_table_entry (COMMA create_table_entry)* COMMA? RPAREN
+create_table_stmt: CREATE (OR REPLACE)? (TABLE | TABLESTORE | EXTERNAL TABLE) (IF NOT EXISTS)? simple_table_ref LPAREN create_table_entry (COMMA create_table_entry)* COMMA? RPAREN
table_inherits?
table_partition_by?
with_table_settings?
diff --git a/ydb/library/yql/sql/v1/format/sql_format.cpp b/ydb/library/yql/sql/v1/format/sql_format.cpp
index f6a47b4485..1e858be6b0 100644
--- a/ydb/library/yql/sql/v1/format/sql_format.cpp
+++ b/ydb/library/yql/sql/v1/format/sql_format.cpp
@@ -891,27 +891,24 @@ private:
Visit(msg.GetToken1());
Visit(msg.GetBlock2());
Visit(msg.GetBlock3());
- Visit(msg.GetRule_simple_table_ref4());
- Visit(msg.GetToken5());
+ Visit(msg.GetBlock4());
+ Visit(msg.GetRule_simple_table_ref5());
+ Visit(msg.GetToken6());
PushCurrentIndent();
NewLine();
- Visit(msg.GetRule_create_table_entry6());
- for (const auto& b : msg.GetBlock7()) {
+ Visit(msg.GetRule_create_table_entry7());
+ for (const auto& b : msg.GetBlock8()) {
Visit(b.GetToken1());
NewLine();
Visit(b.GetRule_create_table_entry2());
}
- if (msg.HasBlock8()) {
- Visit(msg.GetBlock8());
+ if (msg.HasBlock9()) {
+ Visit(msg.GetBlock9());
}
PopCurrentIndent();
NewLine();
- Visit(msg.GetToken9());
- if (msg.HasBlock10()) {
- NewLine();
- Visit(msg.GetBlock10());
- }
+ Visit(msg.GetToken10());
if (msg.HasBlock11()) {
NewLine();
Visit(msg.GetBlock11());
@@ -924,6 +921,10 @@ private:
NewLine();
Visit(msg.GetBlock13());
}
+ if (msg.HasBlock14()) {
+ NewLine();
+ Visit(msg.GetBlock14());
+ }
}
void VisitDropTable(const TRule_drop_table_stmt& msg) {
diff --git a/ydb/library/yql/sql/v1/format/sql_format_ut.cpp b/ydb/library/yql/sql/v1/format/sql_format_ut.cpp
index b5bfca7ed0..cef8e235cc 100644
--- a/ydb/library/yql/sql/v1/format/sql_format_ut.cpp
+++ b/ydb/library/yql/sql/v1/format/sql_format_ut.cpp
@@ -354,6 +354,8 @@ Y_UNIT_TEST_SUITE(CheckSqlFormatter) {
"CREATE EXTERNAL DATA SOURCE usEr WITH (a = \"b\");\n"},
{"creAte exTernAl daTa SouRce if not exists usEr With (a = \"b\")",
"CREATE EXTERNAL DATA SOURCE IF NOT EXISTS usEr WITH (a = \"b\");\n"},
+ {"creAte oR rePlaCe exTernAl daTa SouRce usEr With (a = \"b\")",
+ "CREATE OR REPLACE EXTERNAL DATA SOURCE usEr WITH (a = \"b\");\n"},
{"create external data source eds with (a=\"a\",b=\"b\",c = true)",
"CREATE EXTERNAL DATA SOURCE eds WITH (\n\ta = \"a\",\n\tb = \"b\",\n\tc = TRUE\n);\n"},
{"alter external data source eds set a true, reset (b, c), set (x=y, z=false)",
@@ -388,6 +390,8 @@ Y_UNIT_TEST_SUITE(CheckSqlFormatter) {
TCases cases = {
{"creAte exTernAl TabLe usEr (a int) With (a = \"b\")",
"CREATE EXTERNAL TABLE usEr (\n\ta int\n)\nWITH (a = \"b\");\n"},
+ {"creAte oR rePlaCe exTernAl TabLe usEr (a int) With (a = \"b\")",
+ "CREATE OR REPLACE EXTERNAL TABLE usEr (\n\ta int\n)\nWITH (a = \"b\");\n"},
{"creAte exTernAl TabLe iF NOt Exists usEr (a int) With (a = \"b\")",
"CREATE EXTERNAL TABLE IF NOT EXISTS usEr (\n\ta int\n)\nWITH (a = \"b\");\n"},
{"create external table user (a int) with (a=\"b\",c=\"d\")",
diff --git a/ydb/library/yql/sql/v1/node.h b/ydb/library/yql/sql/v1/node.h
index 83c5d9972e..8bb0ba2030 100644
--- a/ydb/library/yql/sql/v1/node.h
+++ b/ydb/library/yql/sql/v1/node.h
@@ -1231,7 +1231,7 @@ namespace NSQLTranslationV1 {
TNodePtr BuildUpsertObjectOperation(TPosition pos, const TString& objectId, const TString& typeId,
std::map<TString, TDeferredAtom>&& features, const TObjectOperatorContext& context);
TNodePtr BuildCreateObjectOperation(TPosition pos, const TString& objectId, const TString& typeId,
- bool existingOk, std::map<TString, TDeferredAtom>&& features, const TObjectOperatorContext& context);
+ bool existingOk, bool replaceIfExists, std::map<TString, TDeferredAtom>&& features, const TObjectOperatorContext& context);
TNodePtr BuildAlterObjectOperation(TPosition pos, const TString& secretId, const TString& typeId,
std::map<TString, TDeferredAtom>&& features, std::set<TString>&& featuresToReset, const TObjectOperatorContext& context);
TNodePtr BuildDropObjectOperation(TPosition pos, const TString& secretId, const TString& typeId,
diff --git a/ydb/library/yql/sql/v1/object_processing.h b/ydb/library/yql/sql/v1/object_processing.h
index e33ed5f84d..4114235ee6 100644
--- a/ydb/library/yql/sql/v1/object_processing.h
+++ b/ydb/library/yql/sql/v1/object_processing.h
@@ -40,19 +40,30 @@ private:
std::set<TString> FeaturesToReset;
protected:
bool ExistingOk = false;
+ bool ReplaceIfExists = false;
protected:
virtual INode::TPtr BuildOptions() const override {
- return Y(Q(Y(Q("mode"), Q(ExistingOk ? "createObjectIfNotExists" : "createObject"))));
+ TString mode;
+ if (ExistingOk) {
+ mode = "createObjectIfNotExists";
+ } else if (ReplaceIfExists) {
+ mode = "createObjectOrReplace";
+ } else {
+ mode = "createObject";
+ }
+
+ return Y(Q(Y(Q("mode"), Q(mode))));
}
virtual INode::TPtr FillFeatures(INode::TPtr options) const override;
public:
TCreateObject(TPosition pos, const TString& objectId,
- const TString& typeId, bool existingOk, std::map<TString, TDeferredAtom>&& features, std::set<TString>&& featuresToReset, const TObjectOperatorContext& context)
+ const TString& typeId, bool existingOk, bool replaceIfExists, std::map<TString, TDeferredAtom>&& features, std::set<TString>&& featuresToReset, const TObjectOperatorContext& context)
: TBase(pos, objectId, typeId, context)
, Features(std::move(features))
, FeaturesToReset(std::move(featuresToReset))
- , ExistingOk(existingOk) {
- }
+ , ExistingOk(existingOk)
+ , ReplaceIfExists(replaceIfExists) {
+ }
};
class TUpsertObject final: public TCreateObject {
diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp
index 7baae87545..4ea558ead1 100644
--- a/ydb/library/yql/sql/v1/query.cpp
+++ b/ydb/library/yql/sql/v1/query.cpp
@@ -808,11 +808,12 @@ TNodePtr BuildInputTables(TPosition pos, const TTableList& tables, bool inSubque
class TCreateTableNode final: public TAstListNode {
public:
- TCreateTableNode(TPosition pos, const TTableRef& tr, bool existingOk, const TCreateTableParameters& params, TScopedStatePtr scoped)
+ TCreateTableNode(TPosition pos, const TTableRef& tr, bool existingOk, bool replaceIfExists, const TCreateTableParameters& params, TScopedStatePtr scoped)
: TAstListNode(pos)
, Table(tr)
, Params(params)
, ExistingOk(existingOk)
+ , ReplaceIfExists(replaceIfExists)
, Scoped(scoped)
{
scoped->UseCluster(Table.Service, Table.Cluster);
@@ -898,7 +899,13 @@ public:
opts = Table.Options;
}
- opts = L(opts, Q(Y(Q("mode"), Q(ExistingOk ? "create_if_not_exists" : "create"))));
+ if (ExistingOk) {
+ opts = L(opts, Q(Y(Q("mode"), Q("create_if_not_exists"))));
+ } else if (ReplaceIfExists) {
+ opts = L(opts, Q(Y(Q("mode"), Q("create_or_replace"))));
+ } else {
+ opts = L(opts, Q(Y(Q("mode"), Q("create"))));
+ }
THashSet<TString> columnFamilyNames;
@@ -1149,12 +1156,13 @@ private:
const TTableRef Table;
const TCreateTableParameters Params;
const bool ExistingOk;
+ const bool ReplaceIfExists;
TScopedStatePtr Scoped;
};
-TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, bool existingOk, const TCreateTableParameters& params, TScopedStatePtr scoped)
+TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, bool existingOk, bool replaceIfExists, const TCreateTableParameters& params, TScopedStatePtr scoped)
{
- return new TCreateTableNode(pos, tr, existingOk, params, scoped);
+ return new TCreateTableNode(pos, tr, existingOk, replaceIfExists, params, scoped);
}
class TAlterTableNode final: public TAstListNode {
@@ -2109,21 +2117,21 @@ private:
TNodePtr BuildUpsertObjectOperation(TPosition pos, const TString& objectId, const TString& typeId,
std::map<TString, TDeferredAtom>&& features, const TObjectOperatorContext& context) {
- return new TUpsertObject(pos, objectId, typeId, false, std::move(features), std::set<TString>(), context);
+ return new TUpsertObject(pos, objectId, typeId, false, false, std::move(features), std::set<TString>(), context);
}
TNodePtr BuildCreateObjectOperation(TPosition pos, const TString& objectId, const TString& typeId,
- bool existingOk, std::map<TString, TDeferredAtom>&& features, const TObjectOperatorContext& context) {
- return new TCreateObject(pos, objectId, typeId, existingOk, std::move(features), std::set<TString>(), context);
+ bool existingOk, bool replaceIfExists, std::map<TString, TDeferredAtom>&& features, const TObjectOperatorContext& context) {
+ return new TCreateObject(pos, objectId, typeId, existingOk, replaceIfExists, std::move(features), std::set<TString>(), context);
}
TNodePtr BuildAlterObjectOperation(TPosition pos, const TString& secretId, const TString& typeId,
std::map<TString, TDeferredAtom>&& features, std::set<TString>&& featuresToReset, const TObjectOperatorContext& context)
{
- return new TAlterObject(pos, secretId, typeId, false, std::move(features), std::move(featuresToReset), context);
+ return new TAlterObject(pos, secretId, typeId, false, false, std::move(features), std::move(featuresToReset), context);
}
TNodePtr BuildDropObjectOperation(TPosition pos, const TString& secretId, const TString& typeId,
bool missingOk, std::map<TString, TDeferredAtom>&& options, const TObjectOperatorContext& context)
{
- return new TDropObject(pos, secretId, typeId, missingOk, std::move(options), std::set<TString>(), context);
+ return new TDropObject(pos, secretId, typeId, missingOk, false, std::move(options), std::set<TString>(), context);
}
TNodePtr BuildDropRoles(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TVector<TDeferredAtom>& toDrop, bool isUser, bool missingOk, TScopedStatePtr scoped) {
diff --git a/ydb/library/yql/sql/v1/source.h b/ydb/library/yql/sql/v1/source.h
index 4e2489d976..f1a91dec12 100644
--- a/ydb/library/yql/sql/v1/source.h
+++ b/ydb/library/yql/sql/v1/source.h
@@ -295,7 +295,7 @@ namespace NSQLTranslationV1 {
TNodePtr BuildTopicKey(TPosition pos, const TDeferredAtom& cluster, const TDeferredAtom& name);
TNodePtr BuildInputOptions(TPosition pos, const TTableHints& hints);
TNodePtr BuildInputTables(TPosition pos, const TTableList& tables, bool inSubquery, TScopedStatePtr scoped);
- TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, bool existingOk, const TCreateTableParameters& params, TScopedStatePtr scoped);
+ TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, bool existingOk, bool replaceIfExists, const TCreateTableParameters& params, TScopedStatePtr scoped);
TNodePtr BuildAlterTable(TPosition pos, const TTableRef& tr, const TAlterTableParameters& params, TScopedStatePtr scoped);
TNodePtr BuildDropTable(TPosition pos, const TTableRef& table, bool missingOk, ETableType tableType, TScopedStatePtr scoped);
TNodePtr BuildWriteTable(TPosition pos, const TString& label, const TTableRef& table, EWriteColumnMode mode, TNodePtr options,
diff --git a/ydb/library/yql/sql/v1/sql_query.cpp b/ydb/library/yql/sql/v1/sql_query.cpp
index 931614d1dc..79f9c34696 100644
--- a/ydb/library/yql/sql/v1/sql_query.cpp
+++ b/ydb/library/yql/sql/v1/sql_query.cpp
@@ -154,7 +154,17 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
case TRule_sql_stmt_core::kAltSqlStmtCore4: {
Ctx.BodyPart();
const auto& rule = core.GetAlt_sql_stmt_core4().GetRule_create_table_stmt1();
- const auto& block = rule.GetBlock2();
+
+ bool replaceIfExists = false;
+ if (rule.HasBlock2()) { // OR REPLACE
+ replaceIfExists = true;
+ Y_DEBUG_ABORT_UNLESS(
+ rule.GetBlock2().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_OR &&
+ rule.GetBlock2().GetToken2().GetId() == SQLv1LexerTokens::TOKEN_REPLACE
+ );
+ }
+
+ const auto& block = rule.GetBlock3();
ETableType tableType = ETableType::Table;
if (block.HasAlt2() && block.GetAlt2().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_TABLESTORE) {
tableType = ETableType::TableStore;
@@ -163,57 +173,63 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
}
bool existingOk = false;
- if (rule.HasBlock3()) { // IF NOT EXISTS
+ if (rule.HasBlock4()) { // IF NOT EXISTS
existingOk = true;
Y_DEBUG_ABORT_UNLESS(
- rule.GetBlock3().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_IF &&
- rule.GetBlock3().GetToken2().GetId() == SQLv1LexerTokens::TOKEN_NOT &&
- rule.GetBlock3().GetToken3().GetId() == SQLv1LexerTokens::TOKEN_EXISTS
+ rule.GetBlock4().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_IF &&
+ rule.GetBlock4().GetToken2().GetId() == SQLv1LexerTokens::TOKEN_NOT &&
+ rule.GetBlock4().GetToken3().GetId() == SQLv1LexerTokens::TOKEN_EXISTS
);
}
+ if (replaceIfExists && tableType != ETableType::ExternalTable) {
+ Context().Error(GetPos(rule.GetBlock2().GetToken1()))
+ << "OR REPLACE feature is supported only for EXTERNAL DATA SOURCE and EXTERNAL TABLE";
+ return false;
+ }
+
TTableRef tr;
- if (!SimpleTableRefImpl(rule.GetRule_simple_table_ref4(), tr)) {
+ if (!SimpleTableRefImpl(rule.GetRule_simple_table_ref5(), tr)) {
return false;
}
TCreateTableParameters params{.TableType=tableType};
- if (!CreateTableEntry(rule.GetRule_create_table_entry6(), params)) {
+ if (!CreateTableEntry(rule.GetRule_create_table_entry7(), params)) {
return false;
}
- for (auto& block: rule.GetBlock7()) {
+ for (auto& block: rule.GetBlock8()) {
if (!CreateTableEntry(block.GetRule_create_table_entry2(), params)) {
return false;
}
}
- if (rule.HasBlock10()) {
- Context().Error(GetPos(rule.GetBlock10().GetRule_table_inherits1().GetToken1()))
+ if (rule.HasBlock11()) {
+ Context().Error(GetPos(rule.GetBlock11().GetRule_table_inherits1().GetToken1()))
<< "INHERITS clause is not supported yet";
return false;
}
- if (rule.HasBlock11()) {
+ if (rule.HasBlock12()) {
if (tableType == ETableType::TableStore) {
- Context().Error(GetPos(rule.GetBlock11().GetRule_table_partition_by1().GetToken1()))
+ Context().Error(GetPos(rule.GetBlock12().GetRule_table_partition_by1().GetToken1()))
<< "PARTITION BY is not supported for TABLESTORE";
return false;
}
- const auto list = rule.GetBlock11().GetRule_table_partition_by1().GetRule_pure_column_list4();
+ const auto list = rule.GetBlock12().GetRule_table_partition_by1().GetRule_pure_column_list4();
params.PartitionByColumns.push_back(IdEx(list.GetRule_an_id2(), *this));
for (auto& node : list.GetBlock3()) {
params.PartitionByColumns.push_back(IdEx(node.GetRule_an_id2(), *this));
}
}
- if (rule.HasBlock12()) {
- if (!CreateTableSettings(rule.GetBlock12().GetRule_with_table_settings1(), params)) {
+ if (rule.HasBlock13()) {
+ if (!CreateTableSettings(rule.GetBlock13().GetRule_with_table_settings1(), params)) {
return false;
}
}
- if (rule.HasBlock13()) {
- Context().Error(GetPos(rule.GetBlock13().GetRule_table_tablestore1().GetToken1()))
+ if (rule.HasBlock14()) {
+ Context().Error(GetPos(rule.GetBlock14().GetRule_table_tablestore1().GetToken1()))
<< "TABLESTORE clause is not supported yet";
return false;
}
@@ -222,7 +238,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
return false;
}
- AddStatementToBlocks(blocks, BuildCreateTable(Ctx.Pos(), tr, existingOk, params, Ctx.Scoped));
+ AddStatementToBlocks(blocks, BuildCreateTable(Ctx.Pos(), tr, existingOk, replaceIfExists, params, Ctx.Scoped));
break;
}
case TRule_sql_stmt_core::kAltSqlStmtCore5: {
@@ -677,7 +693,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
}
}
- AddStatementToBlocks(blocks, BuildCreateObjectOperation(Ctx.Pos(), objectId, typeId, existingOk, std::move(kv), context));
+ AddStatementToBlocks(blocks, BuildCreateObjectOperation(Ctx.Pos(), objectId, typeId, existingOk, false, std::move(kv), context));
break;
}
case TRule_sql_stmt_core::kAltSqlStmtCore28: {
@@ -734,33 +750,42 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
break;
}
case TRule_sql_stmt_core::kAltSqlStmtCore30: {
- // create_external_data_source_stmt: CREATE EXTERNAL DATA SOURCE (IF NOT EXISTS)? name WITH (k=v,...);
+ // create_external_data_source_stmt: CREATE (OR REPLACE)? EXTERNAL DATA SOURCE (IF NOT EXISTS)? name WITH (k=v,...);
auto& node = core.GetAlt_sql_stmt_core30().GetRule_create_external_data_source_stmt1();
TObjectOperatorContext context(Ctx.Scoped);
- if (node.GetRule_object_ref6().HasBlock1()) {
- if (!ClusterExpr(node.GetRule_object_ref6().GetBlock1().GetRule_cluster_expr1(),
+ if (node.GetRule_object_ref7().HasBlock1()) {
+ if (!ClusterExpr(node.GetRule_object_ref7().GetBlock1().GetRule_cluster_expr1(),
false, context.ServiceId, context.Cluster)) {
return false;
}
}
+ bool replaceIfExists = false;
+ if (node.HasBlock2()) { // OR REPLACE
+ replaceIfExists = true;
+ Y_DEBUG_ABORT_UNLESS(
+ node.GetBlock2().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_OR &&
+ node.GetBlock2().GetToken2().GetId() == SQLv1LexerTokens::TOKEN_REPLACE
+ );
+ }
+
bool existingOk = false;
- if (node.HasBlock5()) { // IF NOT EXISTS
+ if (node.HasBlock6()) { // IF NOT EXISTS
existingOk = true;
Y_DEBUG_ABORT_UNLESS(
- node.GetBlock5().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_IF &&
- node.GetBlock5().GetToken2().GetId() == SQLv1LexerTokens::TOKEN_NOT &&
- node.GetBlock5().GetToken3().GetId() == SQLv1LexerTokens::TOKEN_EXISTS
+ node.GetBlock6().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_IF &&
+ node.GetBlock6().GetToken2().GetId() == SQLv1LexerTokens::TOKEN_NOT &&
+ node.GetBlock6().GetToken3().GetId() == SQLv1LexerTokens::TOKEN_EXISTS
);
}
- const TString& objectId = Id(node.GetRule_object_ref6().GetRule_id_or_at2(), *this).second;
+ const TString& objectId = Id(node.GetRule_object_ref7().GetRule_id_or_at2(), *this).second;
std::map<TString, TDeferredAtom> kv;
- if (!ParseExternalDataSourceSettings(kv, node.GetRule_with_table_settings7())) {
+ if (!ParseExternalDataSourceSettings(kv, node.GetRule_with_table_settings8())) {
return false;
}
- AddStatementToBlocks(blocks, BuildCreateObjectOperation(Ctx.Pos(), BuildTablePath(Ctx.GetPrefixPath(context.ServiceId, context.Cluster), objectId), "EXTERNAL_DATA_SOURCE", existingOk, std::move(kv), context));
+ AddStatementToBlocks(blocks, BuildCreateObjectOperation(Ctx.Pos(), BuildTablePath(Ctx.GetPrefixPath(context.ServiceId, context.Cluster), objectId), "EXTERNAL_DATA_SOURCE", existingOk, replaceIfExists, std::move(kv), context));
break;
}
case TRule_sql_stmt_core::kAltSqlStmtCore31: {
@@ -1085,6 +1110,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
BuildTablePath(Ctx.GetPrefixPath(context.ServiceId, context.Cluster), objectId),
TypeId,
false,
+ false,
std::move(features),
context));
break;
diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp
index 178f2a7996..6da68c791b 100644
--- a/ydb/library/yql/sql/v1/sql_ut.cpp
+++ b/ydb/library/yql/sql/v1/sql_ut.cpp
@@ -5582,6 +5582,50 @@ Y_UNIT_TEST_SUITE(ExternalDataSource) {
UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]);
}
+ Y_UNIT_TEST(CreateExternalDataSourceOrReplace) {
+ NYql::TAstParseResult res = SqlToYql(R"(
+ USE plato;
+ CREATE OR REPLACE EXTERNAL DATA SOURCE MyDataSource WITH (
+ SOURCE_TYPE="ObjectStorage",
+ LOCATION="my-bucket",
+ AUTH_METHOD="NONE"
+ );
+ )");
+ UNIT_ASSERT(res.Root);
+
+ TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) {
+ if (word == "Write") {
+ UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"NONE") '('"location" '"my-bucket") '('"source_type" '"ObjectStorage"))#");
+ UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObjectOrReplace"));
+ }
+ };
+
+ TWordCountHive elementStat = { {TString("Write"), 0} };
+ VerifyProgram(res, elementStat, verifyLine);
+
+ UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]);
+ }
+
+ Y_UNIT_TEST(CreateOrReplaceForUnsupportedTableTypesShouldFail) {
+ ExpectFailWithError(R"sql(
+ USE plato;
+ CREATE OR REPLACE TABLE t (a int32 not null, primary key(a, a));
+ )sql" , "<main>:3:23: Error: OR REPLACE feature is supported only for EXTERNAL DATA SOURCE and EXTERNAL TABLE\n");
+
+ ExpectFailWithError(R"sql(
+ USE plato;
+ CREATE OR REPLACE TABLE t (
+ Key Uint64,
+ Value1 String,
+ PRIMARY KEY (Key)
+ )
+ WITH (
+ STORE = COLUMN,
+ AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10
+ );
+ )sql" , "<main>:3:23: Error: OR REPLACE feature is supported only for EXTERNAL DATA SOURCE and EXTERNAL TABLE\n");
+ }
+
Y_UNIT_TEST(CreateExternalDataSourceWithBadArguments) {
ExpectFailWithError(R"sql(
USE plato;
@@ -5913,6 +5957,31 @@ Y_UNIT_TEST_SUITE(ExternalTable) {
UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]);
}
+ Y_UNIT_TEST(CreateExternalTableOrReplace) {
+ NYql::TAstParseResult res = SqlToYql(R"(
+ USE plato;
+ CREATE OR REPLACE EXTERNAL TABLE mytable (
+ a int
+ ) WITH (
+ DATA_SOURCE="/Root/mydatasource",
+ LOCATION="/folder1/*"
+ );
+ )");
+ UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString());
+
+ TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) {
+ if (word == "Write") {
+ UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('data_source_path (String '"/Root/mydatasource")) '('location (String '"/folder1/*")))) '('tableType 'externalTable)))))#");
+ UNIT_ASSERT_STRING_CONTAINS(line, "create_or_replace");
+ }
+ };
+
+ TWordCountHive elementStat = { {TString("Write"), 0} };
+ VerifyProgram(res, elementStat, verifyLine);
+
+ UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]);
+ }
+
Y_UNIT_TEST(AlterExternalTableAddColumn) {
NYql::TAstParseResult res = SqlToYql(R"sql(
USE plato;
@@ -6230,7 +6299,7 @@ Y_UNIT_TEST_SUITE(TViewSyntaxTest) {
);
UNIT_ASSERT_STRING_CONTAINS(res.Issues.ToString(), "SECURITY_INVOKER option must be explicitly enabled");
}
-
+
Y_UNIT_TEST(CreateViewFromTable) {
constexpr const char* path = "/PathPrefix/TheView";
constexpr const char* query = R"(
diff --git a/ydb/services/metadata/abstract/parsing.h b/ydb/services/metadata/abstract/parsing.h
index 49c882a2ab..6e6743327b 100644
--- a/ydb/services/metadata/abstract/parsing.h
+++ b/ydb/services/metadata/abstract/parsing.h
@@ -13,6 +13,7 @@ namespace NYql {
namespace NObjectOptionsParsing {
Y_HAS_MEMBER(ExistingOk); // for create
Y_HAS_MEMBER(MissingOk); // for drop
+Y_HAS_MEMBER(ReplaceIfExists); // for create
} // namespace NObjectOptionsParsing
class TObjectSettingsImpl {
@@ -24,6 +25,7 @@ private:
YDB_READONLY_DEF(TString, ObjectId);
YDB_READONLY_DEF(bool, ExistingOk); // for create
YDB_READONLY_DEF(bool, MissingOk); // for drop
+ YDB_READONLY_DEF(bool, ReplaceIfExists); // for create
TFeatures Features;
std::shared_ptr<TFeaturesExtractor> FeaturesExtractor;
public:
@@ -45,6 +47,9 @@ public:
bool DeserializeFromKi(const TKiObject& data) {
ObjectId = data.ObjectId();
TypeId = data.TypeId();
+ if constexpr (NObjectOptionsParsing::THasReplaceIfExists<TKiObject>::value) {
+ ReplaceIfExists = (data.ReplaceIfExists().Value() == "1");
+ }
if constexpr (NObjectOptionsParsing::THasExistingOk<TKiObject>::value) {
ExistingOk = (data.ExistingOk().Value() == "1");
}