diff options
author | hcpp <hcpp@ydb.tech> | 2023-03-14 12:42:01 +0300 |
---|---|---|
committer | hcpp <hcpp@ydb.tech> | 2023-03-14 12:42:01 +0300 |
commit | 756a863bc1f7536e8dd5bbde38b00f6afb851d1e (patch) | |
tree | d501e0c728ee3e981b9a3e4e41924eddf3d1c2a0 | |
parent | 6b69664787646ae72170e8444fefc6d25fd57c13 (diff) | |
download | ydb-756a863bc1f7536e8dd5bbde38b00f6afb851d1e.tar.gz |
external table has been added to yql/kqp
-rw-r--r-- | ydb/core/kqp/gateway/kqp_ic_gateway.cpp | 2 | ||||
-rw-r--r-- | ydb/core/kqp/provider/yql_kikimr_datasink.cpp | 7 | ||||
-rw-r--r-- | ydb/core/kqp/provider/yql_kikimr_exec.cpp | 154 | ||||
-rw-r--r-- | ydb/core/kqp/provider/yql_kikimr_gateway.cpp | 8 | ||||
-rw-r--r-- | ydb/core/kqp/provider/yql_kikimr_gateway.h | 8 | ||||
-rw-r--r-- | ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp | 9 | ||||
-rw-r--r-- | ydb/core/kqp/provider/yql_kikimr_type_ann.cpp | 91 | ||||
-rw-r--r-- | ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp | 284 | ||||
-rw-r--r-- | ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp | 2 | ||||
-rw-r--r-- | ydb/core/tx/schemeshard/ut_external_data_source.cpp | 2 | ||||
-rw-r--r-- | ydb/library/yql/sql/v1/SQLv1.g.in | 4 | ||||
-rw-r--r-- | ydb/library/yql/sql/v1/format/sql_format.cpp | 2 | ||||
-rw-r--r-- | ydb/library/yql/sql/v1/format/sql_format_ut.cpp | 12 | ||||
-rw-r--r-- | ydb/library/yql/sql/v1/node.h | 12 | ||||
-rw-r--r-- | ydb/library/yql/sql/v1/query.cpp | 40 | ||||
-rw-r--r-- | ydb/library/yql/sql/v1/sql.cpp | 134 | ||||
-rw-r--r-- | ydb/library/yql/sql/v1/sql_ut.cpp | 166 |
17 files changed, 835 insertions, 102 deletions
diff --git a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp index ef64a1442d2..59ea3cbad02 100644 --- a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp +++ b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp @@ -1248,7 +1248,7 @@ public: NKikimrSchemeOp::TExternalTableDescription& externalTableDesc = *schemeTx.MutableCreateExternalTable(); FillCreateExternalTableColumnDesc(externalTableDesc, pathPair.second, settings); - return SendSchemeRequest(ev.Release()); + return SendSchemeRequest(ev.Release(), true); } catch (yexception& e) { return MakeFuture(ResultFromException<TGenericResult>(e)); diff --git a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp index bada9ec91a0..ebbe3af7915 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp @@ -494,11 +494,16 @@ public: YQL_ENSURE(settings.Columns); YQL_ENSURE(!settings.Columns.Cast().Empty()); - if (!settings.PrimaryKey) { + const bool isExternalTable = settings.TableType && settings.TableType.Cast() == "externalTable"; + if (!isExternalTable && !settings.PrimaryKey) { ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Primary key is required for ydb tables.")); return nullptr; } + if (!settings.PrimaryKey.IsValid()) { + settings.PrimaryKey = Build<TCoAtomList>(ctx, node->Pos()).Done(); + } + if (!settings.PartitionBy.IsValid()) { settings.PartitionBy = Build<TCoAtomList>(ctx, node->Pos()).Done(); } diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp index 47ebe3c8ba8..30cf95a1ff7 100644 --- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp @@ -195,6 +195,48 @@ namespace { return out; } + TCreateExternalTableSettings ParseCreateExternalTableSettings(TKiCreateTable create, const TTableSettings& settings) { + TCreateExternalTableSettings out; + out.ExternalTable = TString(create.Table()); + + for (auto item : create.Columns()) { + auto columnTuple = item.Cast<TExprList>(); + auto nameNode = columnTuple.Item(0).Cast<TCoAtom>(); + auto typeNode = columnTuple.Item(1); + + auto columnName = TString(nameNode.Value()); + auto columnType = typeNode.Ref().GetTypeAnn(); + YQL_ENSURE(columnType && columnType->GetKind() == ETypeAnnotationKind::Type); + + auto type = columnType->Cast<TTypeExprType>()->GetType(); + auto notNull = type->GetKind() != ETypeAnnotationKind::Optional; + auto actualType = notNull ? type : type->Cast<TOptionalExprType>()->GetItemType(); + auto dataType = actualType->Cast<TDataExprType>(); + + TKikimrColumnMetadata columnMeta; + columnMeta.Name = columnName; + columnMeta.Type = dataType->GetName(); + columnMeta.NotNull = notNull; + + out.ColumnOrder.push_back(columnName); + out.Columns.insert(std::make_pair(columnName, columnMeta)); + } + if (settings.DataSourcePath) { + out.DataSourcePath = *settings.DataSourcePath; + } + if (settings.Location) { + out.Location = *settings.Location; + } + out.SourceTypeParameters = settings.ExternalSourceParameters; + return out; + } + + TDropExternalTableSettings ParseDropExternalTableSettings(TKiDropTable drop) { + return TDropExternalTableSettings{ + .ExternalTable = TString(drop.Table()) + }; + } + TAlterTableStoreSettings ParseAlterTableStoreSettings(TKiAlterTable alter) { return TAlterTableStoreSettings{ .TableStore = TString(alter.Table()) @@ -602,9 +644,14 @@ public: auto cluster = TString(maybeCreate.Cast().DataSink().Cluster()); auto& table = SessionCtx->Tables().GetTable(cluster, TString(maybeCreate.Cast().Table())); - bool isTableStore = (table.Metadata->TableType == ETableType::TableStore); + auto tableTypeItem = table.Metadata->TableType; + if (tableTypeItem == ETableType::ExternalTable && !SessionCtx->Config().FeatureFlags.GetEnableExternalDataSources()) { + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), + TStringBuilder() << "External table are disabled. Please contact your system administrator to enable it")); + return SyncError(); + } - if (!isTableStore && !ApplyDdlOperation(cluster, input->Pos(), table.Metadata->Name, TYdbOperation::CreateTable, ctx)) { + if ((tableTypeItem == ETableType::Unknown || tableTypeItem == ETableType::Table) && !ApplyDdlOperation(cluster, input->Pos(), table.Metadata->Name, TYdbOperation::CreateTable, ctx)) { return SyncError(); } @@ -613,19 +660,27 @@ public: future = CreateDummySuccess(); } else { bool isColumn = (table.Metadata->StoreType == EStoreType::Column); - - if (isTableStore) { - if (!isColumn) { - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), - TStringBuilder() << "TABLESTORE with not COLUMN store")); - return SyncError(); + switch (tableTypeItem) { + case ETableType::ExternalTable: { + future = Gateway->CreateExternalTable(cluster, + ParseCreateExternalTableSettings(maybeCreate.Cast(), table.Metadata->TableSettings), false); + break; + } + case ETableType::TableStore: { + if (!isColumn) { + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), + TStringBuilder() << "TABLESTORE with not COLUMN store")); + return SyncError(); + } + future = Gateway->CreateTableStore(cluster, + ParseCreateTableStoreSettings(maybeCreate.Cast(), table.Metadata->TableSettings)); + break; + } + case ETableType::Table: + case ETableType::Unknown: { + future = isColumn ? Gateway->CreateColumnTable(table.Metadata, true) : Gateway->CreateTable(table.Metadata, true); + break; } - future = Gateway->CreateTableStore(cluster, - ParseCreateTableStoreSettings(maybeCreate.Cast(), table.Metadata->TableSettings)); - } else if (isColumn) { - future = Gateway->CreateColumnTable(table.Metadata, true); - } else { - future = Gateway->CreateTable(table.Metadata, true); } } @@ -634,7 +689,7 @@ public: Y_UNUSED(res); auto resultNode = ctx.NewWorld(input->Pos()); return resultNode; - }, isTableStore ? "Executing CREATE TABLESTORE" : "Executing CREATE TABLE"); + }, GetCreateTableDebugString(tableTypeItem)); } if (auto maybeDrop = TMaybeNode<TKiDropTable>(input)) { @@ -649,39 +704,56 @@ public: auto cluster = TString(maybeDrop.Cast().DataSink().Cluster()); TString tableName = TString(maybeDrop.Cast().Table()); auto& table = SessionCtx->Tables().GetTable(cluster, tableName); - auto tableType = TString(maybeDrop.Cast().TableType()); - bool isTableStore; - - switch (GetTableTypeFromString(tableType)) { + auto tableTypeString = TString(maybeDrop.Cast().TableType()); + auto tableTypeItem = GetTableTypeFromString(tableTypeString); + switch (tableTypeItem) { case ETableType::Table: if (!ApplyDdlOperation(cluster, input->Pos(), tableName, TYdbOperation::DropTable, ctx)) { return SyncError(); } - isTableStore = false; break; case ETableType::TableStore: - isTableStore = true; + case ETableType::ExternalTable: break; case ETableType::Unknown: - default: - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Unsupported table type " << tableType)); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Unsupported table type " << tableTypeString)); return SyncError(); } + if (tableTypeItem == ETableType::ExternalTable && !SessionCtx->Config().FeatureFlags.GetEnableExternalDataSources()) { + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), + TStringBuilder() << "External table are disabled. Please contact your system administrator to enable it")); + return SyncError(); + } + bool prepareOnly = SessionCtx->Query().PrepareOnly; - auto future = prepareOnly ? CreateDummySuccess() : ( - isTableStore - ? Gateway->DropTableStore(cluster, ParseDropTableStoreSettings(maybeDrop.Cast())) - : Gateway->DropTable(table.Metadata->Cluster, table.Metadata->Name) - ); + NThreading::TFuture<IKikimrGateway::TGenericResult> future; + if (prepareOnly) { + future = CreateDummySuccess(); + } else { + switch (tableTypeItem) { + case ETableType::Table: + future = Gateway->DropTable(table.Metadata->Cluster, table.Metadata->Name); + break; + case ETableType::TableStore: + future = Gateway->DropTableStore(cluster, ParseDropTableStoreSettings(maybeDrop.Cast())); + break; + case ETableType::ExternalTable: + future = Gateway->DropExternalTable(cluster, ParseDropExternalTableSettings(maybeDrop.Cast())); + break; + case ETableType::Unknown: + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Unsupported table type " << tableTypeString)); + return SyncError(); + } + } return WrapFuture(future, [](const IKikimrGateway::TGenericResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { Y_UNUSED(res); auto resultNode = ctx.NewWorld(input->Pos()); return resultNode; - }, isTableStore ? "Executing DROP TABLESTORE" : "Executing DROP TABLE"); + }, GetDropTableDebugString(tableTypeItem)); input->SetState(TExprNode::EState::ExecutionComplete); input->SetResult(ctx.NewWorld(input->Pos())); @@ -1318,6 +1390,30 @@ private: using TExecuteFinalizeFunc = std::function<void(TExprBase node, const IKikimrQueryExecutor::TQueryResult& result, TExprContext& ctx)>; + static TString GetCreateTableDebugString(ETableType tableType) { + switch (tableType) { + case ETableType::ExternalTable: + return "Executing CREATE EXTERNAL TABLE"; + case ETableType::TableStore: + return "Executing CREATE TABLESTORE"; + case ETableType::Table: + case ETableType::Unknown: + return "Executing CREATE TABLE"; + } + } + + static TString GetDropTableDebugString(ETableType tableType) { + switch (tableType) { + case ETableType::ExternalTable: + return "Executing DROP EXTERNAL TABLE"; + case ETableType::TableStore: + return "Executing DROP TABLESTORE"; + case ETableType::Table: + case ETableType::Unknown: + return "Executing DROP TABLE"; + } + } + std::pair<IGraphTransformer::TStatus, TAsyncTransformCallbackFuture> PerformExecution(TExprBase node, TExprContext& ctx, const TString& cluster, TMaybe<TString> mode, const TExecuteRunFunc& runFunc, const TExecuteFinalizeFunc& finalizeFunc) diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp index 5bc3dc8fa24..b31d41a2b78 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp @@ -172,7 +172,7 @@ bool TTtlSettings::TryParse(const NNodes::TCoNameValueTupleList& node, TTtlSetti bool TTableSettings::IsSet() const { return CompactionPolicy || PartitionBy || AutoPartitioningBySize || UniformPartitions || PartitionAtKeys || PartitionSizeMb || AutoPartitioningByLoad || MinPartitions || MaxPartitions || KeyBloomFilter - || ReadReplicasSettings || TtlSettings; + || ReadReplicasSettings || TtlSettings || DataSourcePath || Location || ExternalSourceParameters; } EYqlIssueCode YqlStatusFromYdbStatus(ui32 ydbStatus) { @@ -313,9 +313,13 @@ Ydb::FeatureFlag::Status GetFlagValue(const TMaybe<bool>& value) { ETableType GetTableTypeFromString(const TStringBuf& tableType) { if (tableType == "table") { return ETableType::Table; - } else if (tableType == "tableStore") { + } + if (tableType == "tableStore") { return ETableType::TableStore; } + if (tableType == "externalTable") { + return ETableType::ExternalTable; + } return ETableType::Unknown; } diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index f46147b1d07..d9f5e7d5279 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -173,6 +173,11 @@ struct TTableSettings { TResetableSetting<TString, void> Tiering; TMaybe<TString> PartitionByHashFunction; + // These parameters are only used for external sources + TMaybe<TString> DataSourcePath; + TMaybe<TString> Location; + TVector<std::pair<TString, TString>> ExternalSourceParameters; + bool IsSet() const; }; @@ -283,7 +288,8 @@ enum class EKikimrTableKind : ui32 { enum class ETableType : ui32 { Unknown = 0, Table = 1, - TableStore = 2 + TableStore = 2, + ExternalTable = 3 }; ETableType GetTableTypeFromString(const TStringBuf& tableType); diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp index bb7aa9ce1de..1340464e725 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp @@ -287,7 +287,7 @@ void TestCreateExternalDataSource(TTestActorRuntime& runtime, TIntrusivePtr<IKik UNIT_ASSERT(externalDataSource.ExternalDataSourceInfo->Description.GetAuth().HasNone()); } -void TestCreateExternalTable(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGateway> gateway, const TString& path) { +void TestCreateExternalTable(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGateway> gateway, const TString& path, bool fail = false) { NYql::TCreateExternalTableSettings settings; settings.ExternalTable = path; @@ -305,6 +305,11 @@ void TestCreateExternalTable(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGa auto response = responseFuture.GetValue(); response.Issues().PrintTo(Cerr); + if (fail) { + UNIT_ASSERT_C(!response.Success(), response.Issues().ToString()); + return; + } + UNIT_ASSERT_C(response.Success(), response.Issues().ToString()); auto externalTableDesc = Navigate(runtime, runtime.AllocateEdgeActor(), path, NSchemeCache::TSchemeCacheNavigate::EOp::OpUnknown); @@ -419,7 +424,7 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) { TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); TestCreateExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source"); TestCreateExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table"); - TestCreateExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table"); + TestCreateExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table", true); } Y_UNIT_TEST(TestDropExternalTable) { diff --git a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp index 91dc744539c..9131d8f089f 100644 --- a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp @@ -532,7 +532,7 @@ private: return TStatus::Ok; } - virtual TStatus HandleCreateTable(TKiCreateTable create, TExprContext& ctx) override { +virtual TStatus HandleCreateTable(TKiCreateTable create, TExprContext& ctx) override { TString cluster = TString(create.DataSink().Cluster()); TString table = TString(create.Table()); TString tableType = TString(create.TableType()); @@ -696,6 +696,73 @@ private: } } + switch (meta->TableType) { + case ETableType::Unknown: + case ETableType::TableStore: + case ETableType::Table: { + auto status = FillTableSettings(create, ctx, meta); + if (status != TStatus::Ok) { + return status; + } + break; + } + case ETableType::ExternalTable: { + auto status = FillExternalTableSettings(create, ctx, meta); + if (status != TStatus::Ok) { + return status; + } + break; + } + }; + + if (meta->TableType == ETableType::TableStore && meta->StoreType != EStoreType::Column) { + ctx.AddError(TIssue(ctx.GetPosition(create.Pos()), + TStringBuilder() << "TABLESTORE recuires STORE = COLUMN setting now")); + return TStatus::Error; + } + + auto& tableDesc = SessionCtx->Tables().GetTable(cluster, table); + if (meta->TableType == ETableType::Table && tableDesc.DoesExist() && !tableDesc.Metadata->IsSameTable(*meta)) { + ctx.AddError(TIssue(ctx.GetPosition(create.Pos()), TStringBuilder() + << "Table name conflict: " << NCommon::FullTableName(cluster, table) + << " is used to reference multiple tables.")); + return TStatus::Error; + } + + tableDesc.Metadata = meta; + bool sysColumnsEnabled = SessionCtx->Config().SystemColumnsEnabled(); + YQL_ENSURE(tableDesc.Load(ctx, sysColumnsEnabled)); + + create.Ptr()->SetTypeAnn(create.World().Ref().GetTypeAnn()); + return TStatus::Ok; + } + + TStatus FillExternalTableSettings(TKiCreateTable create, TExprContext& ctx, TKikimrTableMetadataPtr meta) { + for (const auto& setting : create.TableSettings()) { + auto name = setting.Name().Value(); + if (name == "data_source_path") { + meta->TableSettings.DataSourcePath = TString( + setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() + ); + } else if (name == "location") { + meta->TableSettings.Location = TString( + setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() + ); + } else { + meta->TableSettings.ExternalSourceParameters.emplace_back(name, TString( + setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value() + )); + } + } + if (!meta->TableSettings.DataSourcePath) { + ctx.AddError(TIssue(ctx.GetPosition(create.Pos()), + "DATA_SOURCE parameter is required for external table")); + return TStatus::Error; + } + return TStatus::Ok; + } + + TStatus FillTableSettings(TKiCreateTable create, TExprContext& ctx, TKikimrTableMetadataPtr meta) { for (const auto& setting : create.TableSettings()) { auto name = setting.Name().Value(); if (name == "compactionPolicy") { @@ -843,29 +910,11 @@ private: return TStatus::Error; } } - - if (meta->TableType == ETableType::TableStore && meta->StoreType != EStoreType::Column) { - ctx.AddError(TIssue(ctx.GetPosition(create.Pos()), - TStringBuilder() << "TABLESTORE recuires STORE = COLUMN setting now")); - return TStatus::Error; - } - - auto& tableDesc = SessionCtx->Tables().GetTable(cluster, table); - if (meta->TableType == ETableType::Table && tableDesc.DoesExist() && !tableDesc.Metadata->IsSameTable(*meta)) { - ctx.AddError(TIssue(ctx.GetPosition(create.Pos()), TStringBuilder() - << "Table name conflict: " << NCommon::FullTableName(cluster, table) - << " is used to reference multiple tables.")); - return TStatus::Error; - } - - tableDesc.Metadata = meta; - bool sysColumnsEnabled = SessionCtx->Config().SystemColumnsEnabled(); - YQL_ENSURE(tableDesc.Load(ctx, sysColumnsEnabled)); - - create.Ptr()->SetTypeAnn(create.World().Ref().GetTypeAnn()); return TStatus::Ok; } + + virtual TStatus HandleDropTable(TKiDropTable node, TExprContext& ctx) override { auto table = SessionCtx->Tables().EnsureTableExists(TString(node.DataSink().Cluster()), TString(node.Table().Value()), node.Pos(), ctx); if (!table) { diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index 003d2d903b5..699f0d359c5 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -3736,6 +3736,290 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Check failed: path: '/Root/ExternalDataSource', error: path exist"); } } + + Y_UNIT_TEST(CreateExternalTable) { + TKikimrRunner kikimr; + kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString externalDataSourceName = "/Root/ExternalDataSource"; + TString externalTableName = "/Root/ExternalTable"; + auto query = TStringBuilder() << R"( + CREATE EXTERNAL DATA SOURCE `)" << externalDataSourceName << R"(` WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="my-bucket", + AUTH_METHOD="NONE" + ); + CREATE EXTERNAL TABLE `)" << externalTableName << R"(` ( + Key Uint64, + Value String + ) WITH ( + DATA_SOURCE=")" << externalDataSourceName << R"(", + LOCATION="/" + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); + + auto& runtime = *kikimr.GetTestServer().GetRuntime(); + auto externalTableDesc = Navigate(runtime, runtime.AllocateEdgeActor(), externalTableName, NKikimr::NSchemeCache::TSchemeCacheNavigate::EOp::OpUnknown); + const auto& externalTable = externalTableDesc->ResultSet.at(0); + UNIT_ASSERT_EQUAL(externalTable.Kind, NKikimr::NSchemeCache::TSchemeCacheNavigate::EKind::KindExternalTable); + UNIT_ASSERT(externalTable.ExternalTableInfo); + UNIT_ASSERT_VALUES_EQUAL(externalTable.ExternalTableInfo->Description.ColumnsSize(), 2); + UNIT_ASSERT_VALUES_EQUAL(externalTable.ExternalTableInfo->Description.GetDataSourcePath(), externalDataSourceName); + UNIT_ASSERT_VALUES_EQUAL(externalTable.ExternalTableInfo->Description.GetLocation(), "/"); + UNIT_ASSERT_VALUES_EQUAL(externalTable.ExternalTableInfo->Description.GetSourceType(), "ObjectStorage"); + } + + Y_UNIT_TEST(DisableCreateExternalTable) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + auto query = TStringBuilder() << R"( + CREATE EXTERNAL TABLE `/Root/ExternalTable` ( + Key Uint64, + Value String + ) WITH ( + DATA_SOURCE="/Root/ExternalDataSource", + LOCATION="/" + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "External table are disabled. Please contact your system administrator to enable it"); + } + + Y_UNIT_TEST(CreateExternalTableCheckPrimaryKey) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + auto query = TStringBuilder() << R"( + CREATE EXTERNAL TABLE `/Root/ExternalTable` ( + Key Uint64, + Value String, + PRIMARY KEY(Key) + ) WITH ( + DATA_SOURCE="/Root/MyDataSource", + LOCATION="/" + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_UNEQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "PRIMARY KEY is not supported for external table"); + } + + Y_UNIT_TEST(CreateExternalTableValidation) { + TKikimrRunner kikimr; + kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + auto query = TStringBuilder() << R"( + CREATE EXTERNAL TABLE `/Root/ExternalTable` ( + Key Uint64, + Value String, + PRIMARY KEY(Key) + ) WITH ( + LOCATION="/" + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "DATA_SOURCE requires key"); + } + + Y_UNIT_TEST(DropExternalTable) { + TKikimrRunner kikimr; + kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString externalDataSourceName = "/Root/ExternalDataSource"; + TString externalTableName = "/Root/ExternalTable"; + { + auto query = TStringBuilder() << R"( + CREATE EXTERNAL DATA SOURCE `)" << externalDataSourceName << R"(` WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="my-bucket", + AUTH_METHOD="NONE" + ); + CREATE EXTERNAL TABLE `)" << externalTableName << R"(` ( + Key Uint64, + Value String + ) WITH ( + DATA_SOURCE=")" << externalDataSourceName << R"(", + LOCATION="/" + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + auto query = TStringBuilder() << R"( DROP EXTERNAL TABLE `)" << externalTableName << "`"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + auto& runtime = *kikimr.GetTestServer().GetRuntime(); + auto externalTableDesc = Navigate(runtime, runtime.AllocateEdgeActor(), externalTableName, NSchemeCache::TSchemeCacheNavigate::EOp::OpUnknown); + const auto& externalTable = externalTableDesc->ResultSet.at(0); + UNIT_ASSERT_EQUAL(externalTableDesc->ErrorCount, 1); + UNIT_ASSERT_EQUAL(externalTable.Kind, NSchemeCache::TSchemeCacheNavigate::EKind::KindUnknown); + } + + { + auto query = TStringBuilder() << R"( DROP EXTERNAL DATA SOURCE `)" << externalDataSourceName << "`"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + auto& runtime = *kikimr.GetTestServer().GetRuntime(); + auto externalDataSourceDesc = Navigate(runtime, runtime.AllocateEdgeActor(), externalDataSourceName, NSchemeCache::TSchemeCacheNavigate::EOp::OpUnknown); + const auto& externalDataSource = externalDataSourceDesc->ResultSet.at(0); + UNIT_ASSERT_EQUAL(externalDataSourceDesc->ErrorCount, 1); + UNIT_ASSERT_EQUAL(externalDataSource.Kind, NSchemeCache::TSchemeCacheNavigate::EKind::KindUnknown); + } + } + + Y_UNIT_TEST(DisableDropExternalTable) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + auto query = TStringBuilder() << R"( DROP EXTERNAL TABLE `/Root/ExternalDataSource`)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "External table are disabled. Please contact your system administrator to enable it"); + } + + Y_UNIT_TEST(CreateExternalTableWithSettings) { + TKikimrRunner kikimr; + kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString externalDataSourceName = "/Root/ExternalDataSource"; + TString externalTableName = "/Root/ExternalTable"; + auto query = TStringBuilder() << R"( + CREATE EXTERNAL DATA SOURCE `)" << externalDataSourceName << R"(` WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="my-bucket", + AUTH_METHOD="NONE" + ); + CREATE EXTERNAL TABLE `)" << externalTableName << R"(` ( + Key Uint64, + Value String, + year Int, + month Int + ) WITH ( + DATA_SOURCE=")" << externalDataSourceName << R"(", + LOCATION="/folder1/*", + FORMAT="json_as_string", + `projection.enabled`="true", + `projection.year.type`="integer", + `projection.year.min`="2010", + `projection.year.max`="2022", + `projection.year.interval`="1", + `projection.month.type`="integer", + `projection.month.min`="1", + `projection.month.max`="12", + `projection.month.interval`="1", + `projection.month.digits`="2", + `storage.location.template`="${year}/${month}", + PARTITONED_BY = "[year, month]" + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto& runtime = *kikimr.GetTestServer().GetRuntime(); + auto externalTableDesc = Navigate(runtime, runtime.AllocateEdgeActor(), externalTableName, NKikimr::NSchemeCache::TSchemeCacheNavigate::EOp::OpUnknown); + const auto& externalTable = externalTableDesc->ResultSet.at(0); + UNIT_ASSERT_EQUAL(externalTable.Kind, NKikimr::NSchemeCache::TSchemeCacheNavigate::EKind::KindExternalTable); + UNIT_ASSERT(externalTable.ExternalTableInfo); + UNIT_ASSERT_VALUES_EQUAL(externalTable.ExternalTableInfo->Description.ColumnsSize(), 4); + UNIT_ASSERT_VALUES_EQUAL(externalTable.ExternalTableInfo->Description.GetDataSourcePath(), externalDataSourceName); + UNIT_ASSERT_VALUES_EQUAL(externalTable.ExternalTableInfo->Description.GetLocation(), "/folder1/*"); + } + + Y_UNIT_TEST(DoubleCreateExternalTable) { + TKikimrRunner kikimr; + kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString externalDataSourceName = "/Root/ExternalDataSource"; + TString externalTableName = "/Root/ExternalTable"; + { + auto query = TStringBuilder() << R"( + CREATE EXTERNAL DATA SOURCE `)" << externalDataSourceName << R"(` WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="my-bucket", + AUTH_METHOD="NONE" + ); + CREATE EXTERNAL TABLE `)" << externalTableName << R"(` ( + Key Uint64, + Value String + ) WITH ( + DATA_SOURCE=")" << externalDataSourceName << R"(", + LOCATION="/" + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto& runtime = *kikimr.GetTestServer().GetRuntime(); + auto externalTableDesc = Navigate(runtime, runtime.AllocateEdgeActor(), externalTableName, NKikimr::NSchemeCache::TSchemeCacheNavigate::EOp::OpUnknown); + const auto& externalTable = externalTableDesc->ResultSet.at(0); + UNIT_ASSERT_EQUAL(externalTable.Kind, NKikimr::NSchemeCache::TSchemeCacheNavigate::EKind::KindExternalTable); + UNIT_ASSERT(externalTable.ExternalTableInfo); + UNIT_ASSERT_VALUES_EQUAL(externalTable.ExternalTableInfo->Description.ColumnsSize(), 2); + UNIT_ASSERT_VALUES_EQUAL(externalTable.ExternalTableInfo->Description.GetDataSourcePath(), externalDataSourceName); + UNIT_ASSERT_VALUES_EQUAL(externalTable.ExternalTableInfo->Description.GetLocation(), "/"); + UNIT_ASSERT_VALUES_EQUAL(externalTable.ExternalTableInfo->Description.GetSourceType(), "ObjectStorage"); + } + + { + auto query = TStringBuilder() << R"( + CREATE EXTERNAL TABLE `)" << externalTableName << R"(` ( + Key Uint64, + Value String + ) WITH ( + DATA_SOURCE=")" << externalDataSourceName << R"(", + LOCATION="/" + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Check failed: path: '/Root/ExternalTable', error: path exist"); + } + } + + Y_UNIT_TEST(DropDependentExternalDataSource) { + TKikimrRunner kikimr; + kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString externalDataSourceName = "/Root/ExternalDataSource"; + TString externalTableName = "/Root/ExternalTable"; + auto query = TStringBuilder() << R"( + CREATE EXTERNAL DATA SOURCE `)" << externalDataSourceName << R"(` WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="my-bucket", + AUTH_METHOD="NONE" + ); + CREATE EXTERNAL TABLE `)" << externalTableName << R"(` ( + Key Uint64, + Value String + ) WITH ( + DATA_SOURCE=")" << externalDataSourceName << R"(", + LOCATION="/" + );)"; + { + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); + + auto& runtime = *kikimr.GetTestServer().GetRuntime(); + auto externalTableDesc = Navigate(runtime, runtime.AllocateEdgeActor(), externalTableName, NKikimr::NSchemeCache::TSchemeCacheNavigate::EOp::OpUnknown); + const auto& externalTable = externalTableDesc->ResultSet.at(0); + UNIT_ASSERT_EQUAL(externalTable.Kind, NKikimr::NSchemeCache::TSchemeCacheNavigate::EKind::KindExternalTable); + UNIT_ASSERT(externalTable.ExternalTableInfo); + UNIT_ASSERT_VALUES_EQUAL(externalTable.ExternalTableInfo->Description.ColumnsSize(), 2); + UNIT_ASSERT_VALUES_EQUAL(externalTable.ExternalTableInfo->Description.GetDataSourcePath(), externalDataSourceName); + UNIT_ASSERT_VALUES_EQUAL(externalTable.ExternalTableInfo->Description.GetLocation(), "/"); + UNIT_ASSERT_VALUES_EQUAL(externalTable.ExternalTableInfo->Description.GetSourceType(), "ObjectStorage"); + } + + { + auto query = TStringBuilder() << R"( DROP EXTERNAL DATA SOURCE `)" << externalDataSourceName << "`"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Other entities depend on this data source, please remove them at the beginning: /Root/ExternalTable"); + } + } } } // namespace NKqp diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp index cb1af0c455d..f4cc731d9cd 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp @@ -377,7 +377,7 @@ public: context.OnComplete.ActivateTx(OperationId); auto& reference = *externalDataSource->ExternalTableReferences.AddReferences(); - reference.SetPath(externalTableInfo->DataSourcePath); + reference.SetPath(dstPath.PathString()); PathIdFromPathId(externalTable->PathId, reference.MutablePathId()); context.SS->ExternalTables[externalTable->PathId] = externalTableInfo; context.SS->IncrementPathDbRefCount(externalTable->PathId); diff --git a/ydb/core/tx/schemeshard/ut_external_data_source.cpp b/ydb/core/tx/schemeshard/ut_external_data_source.cpp index 9900addb53a..e2719ae0fc5 100644 --- a/ydb/core/tx/schemeshard/ut_external_data_source.cpp +++ b/ydb/core/tx/schemeshard/ut_external_data_source.cpp @@ -335,7 +335,7 @@ Y_UNIT_TEST_SUITE(TExternalDataSourceTest) { TestDropExternalDataSource(runtime, ++txId, "/MyRoot", "ExternalDataSource", - {{NKikimrScheme::StatusSchemeError, "Other entities depend on this data source, please remove them at the beginning: /MyRoot/ExternalDataSource"}}); + {{NKikimrScheme::StatusSchemeError, "Other entities depend on this data source, please remove them at the beginning: /MyRoot/ExternalTable"}}); env.TestWaitNotification(runtime, txId); TestLs(runtime, "/MyRoot/ExternalDataSource", false, NLs::PathExist); diff --git a/ydb/library/yql/sql/v1/SQLv1.g.in b/ydb/library/yql/sql/v1/SQLv1.g.in index 927d94da213..8c2ae80adf6 100644 --- a/ydb/library/yql/sql/v1/SQLv1.g.in +++ b/ydb/library/yql/sql/v1/SQLv1.g.in @@ -489,7 +489,7 @@ object_features: object_feature | LPAREN object_feature (COMMA object_feature)* object_type_ref: an_id_or_type; -create_table_stmt: CREATE (TABLE | TABLESTORE) simple_table_ref LPAREN create_table_entry (COMMA create_table_entry)* RPAREN +create_table_stmt: CREATE (TABLE | TABLESTORE | EXTERNAL TABLE) simple_table_ref LPAREN create_table_entry (COMMA create_table_entry)* RPAREN table_inherits? table_partition_by? with_table_settings? @@ -597,7 +597,7 @@ split_boundaries: literal_value_list: LPAREN literal_value (COMMA literal_value)* RPAREN; -drop_table_stmt: DROP (TABLE | TABLESTORE) (IF EXISTS)? simple_table_ref; +drop_table_stmt: DROP (TABLE | TABLESTORE | EXTERNAL TABLE) (IF EXISTS)? simple_table_ref; create_user_stmt: CREATE USER role_name create_user_option?; alter_user_stmt: ALTER USER role_name (WITH? create_user_option | RENAME TO role_name); diff --git a/ydb/library/yql/sql/v1/format/sql_format.cpp b/ydb/library/yql/sql/v1/format/sql_format.cpp index 501f4beb5e4..90cd1e3cce3 100644 --- a/ydb/library/yql/sql/v1/format/sql_format.cpp +++ b/ydb/library/yql/sql/v1/format/sql_format.cpp @@ -640,7 +640,7 @@ private: PosFromToken(msg.GetToken1()); NewLine(); Visit(msg.GetToken1()); - Visit(msg.GetToken2()); + Visit(msg.GetBlock2()); Visit(msg.GetRule_simple_table_ref3()); Visit(msg.GetToken4()); PushCurrentIndent(); diff --git a/ydb/library/yql/sql/v1/format/sql_format_ut.cpp b/ydb/library/yql/sql/v1/format/sql_format_ut.cpp index 31224099790..41a6b5fd0e5 100644 --- a/ydb/library/yql/sql/v1/format/sql_format_ut.cpp +++ b/ydb/library/yql/sql/v1/format/sql_format_ut.cpp @@ -299,6 +299,18 @@ Y_UNIT_TEST_SUITE(CheckSqlFormatter) { setup.Run(cases); } + Y_UNIT_TEST(ExternalTableOperations) { + TCases cases = { + {"creAte exTernAl TabLe usEr (a int) With (a = \"b\")", + "CREATE EXTERNAL TABLE usEr (\n\ta int\n)\nWITH (a = \"b\");\n\n"}, + {"dRop exTerNal taBlE usEr", + "DROP EXTERNAL TABLE usEr;\n"}, + }; + + TSetup setup; + setup.Run(cases); + } + Y_UNIT_TEST(TypeSelection) { TCases cases = { {"Select tYpe.* frOm Table tYpe", diff --git a/ydb/library/yql/sql/v1/node.h b/ydb/library/yql/sql/v1/node.h index 3cd9dd6c7ec..fb18796a9f7 100644 --- a/ydb/library/yql/sql/v1/node.h +++ b/ydb/library/yql/sql/v1/node.h @@ -75,7 +75,8 @@ namespace NSQLTranslationV1 { enum class ETableType { Table, - TableStore + TableStore, + ExternalTable }; class TContext; @@ -1133,10 +1134,15 @@ namespace NSQLTranslationV1 { TMaybe<TIdentifier> StoreType; TNodePtr PartitionByHashFunction; + TNodePtr DataSourcePath; + TNodePtr Location; + TVector<std::pair<TIdentifier, TNodePtr>> ExternalSourceParameters; + bool IsSet() const { return CompactionPolicy || AutoPartitioningBySize || PartitionSizeMb || AutoPartitioningByLoad || MinPartitions || MaxPartitions || UniformPartitions || PartitionAtKeys || KeyBloomFilter - || ReadReplicasSettings || TtlSettings || Tiering || StoreType || PartitionByHashFunction; + || ReadReplicasSettings || TtlSettings || Tiering || StoreType || PartitionByHashFunction + || DataSourcePath || Location || ExternalSourceParameters; } }; @@ -1397,7 +1403,7 @@ namespace NSQLTranslationV1 { TNodePtr BuildInputTables(TPosition pos, const TTableList& tables, bool inSubquery, TScopedStatePtr scoped); TNodePtr BuildCreateTable(TPosition pos, const TTableRef& tr, const TCreateTableParameters& params, TScopedStatePtr scoped); TNodePtr BuildAlterTable(TPosition pos, const TTableRef& tr, const TAlterTableParameters& params, TScopedStatePtr scoped); - TNodePtr BuildDropTable(TPosition pos, const TTableRef& table, bool isTabletore, TScopedStatePtr scoped); + TNodePtr BuildDropTable(TPosition pos, const TTableRef& table, ETableType tableType, TScopedStatePtr scoped); TNodePtr BuildCreateUser(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TMaybe<TRoleParameters>& params, TScopedStatePtr scoped); TNodePtr BuildCreateGroup(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, TScopedStatePtr scoped); TNodePtr BuildAlterUser(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TRoleParameters& params, TScopedStatePtr scoped); diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp index 737aed2c6ae..63591772b55 100644 --- a/ydb/library/yql/sql/v1/query.cpp +++ b/ydb/library/yql/sql/v1/query.cpp @@ -818,6 +818,15 @@ public: if (Params.TableSettings.IsSet()) { auto settings = Y(); + if (Params.TableSettings.DataSourcePath) { + settings = L(settings, Q(Y(Q("data_source_path"), Params.TableSettings.DataSourcePath))); + } + if (Params.TableSettings.Location) { + settings = L(settings, Q(Y(Q("location"), Params.TableSettings.Location))); + } + for (const auto& item: Params.TableSettings.ExternalSourceParameters) { + settings = L(settings, Q(Y(Q(to_lower(item.first.Name)), item.second))); + } if (Params.TableSettings.CompactionPolicy) { settings = L(settings, Q(Y(Q("compactionPolicy"), Params.TableSettings.CompactionPolicy))); } @@ -890,9 +899,15 @@ public: opts = L(opts, Q(Y(Q("tableSettings"), Q(settings)))); } - - if (Params.TableType == ETableType::TableStore) { - opts = L(opts, Q(Y(Q("tableType"), Q("tableStore")))); + switch (Params.TableType) { + case ETableType::TableStore: + opts = L(opts, Q(Y(Q("tableType"), Q("tableStore")))); + break; + case ETableType::ExternalTable: + opts = L(opts, Q(Y(Q("tableType"), Q("externalTable")))); + break; + case ETableType::Table: + break; } Add("block", Q(Y( @@ -1142,10 +1157,10 @@ TNodePtr BuildAlterTable(TPosition pos, const TTableRef& tr, const TAlterTablePa class TDropTableNode final: public TAstListNode { public: - TDropTableNode(TPosition pos, const TTableRef& tr, bool isTabletore, TScopedStatePtr scoped) + TDropTableNode(TPosition pos, const TTableRef& tr, ETableType tableType, TScopedStatePtr scoped) : TAstListNode(pos) , Table(tr) - , TableType(isTabletore ? ETableType::TableStore : ETableType::Table) + , TableType(tableType) , Scoped(scoped) { FakeSource = BuildFakeSource(pos); @@ -1163,8 +1178,15 @@ public: opts = L(opts, Q(Y(Q("mode"), Q("drop")))); - if (TableType == ETableType::TableStore) { - opts = L(opts, Q(Y(Q("tableType"), Q("tableStore")))); + switch (TableType) { + case ETableType::TableStore: + opts = L(opts, Q(Y(Q("tableType"), Q("tableStore")))); + break; + case ETableType::ExternalTable: + opts = L(opts, Q(Y(Q("tableType"), Q("externalTable")))); + break; + case ETableType::Table: + break; } Add("block", Q(Y( @@ -1186,8 +1208,8 @@ private: TSourcePtr FakeSource; }; -TNodePtr BuildDropTable(TPosition pos, const TTableRef& tr, bool isTabletore, TScopedStatePtr scoped) { - return new TDropTableNode(pos, tr, isTabletore, scoped); +TNodePtr BuildDropTable(TPosition pos, const TTableRef& tr, ETableType tableType, TScopedStatePtr scoped) { + return new TDropTableNode(pos, tr, tableType, scoped); } class TCreateRole final: public TAstListNode { diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp index 4b3d2720511..48025f9dc63 100644 --- a/ydb/library/yql/sql/v1/sql.cpp +++ b/ydb/library/yql/sql/v1/sql.cpp @@ -814,13 +814,15 @@ protected: bool FillFamilySettingsEntry(const TRule_family_settings_entry& settingNode, TFamilyEntry& family); bool FillFamilySettings(const TRule_family_settings& settingsNode, TFamilyEntry& family); - bool CreateTableSettings(const TRule_with_table_settings& settingsNode, TTableSettings& settings); + bool CreateTableSettings(const TRule_with_table_settings& settingsNode, TCreateTableParameters& params); + bool StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value* value, TTableSettings& settings, + ETableType tableType, bool alter, bool reset); bool StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value* value, TTableSettings& settings, bool alter, bool reset); - bool StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value& value, TTableSettings& settings, - bool alter = false); + bool StoreExternalTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value* value, TTableSettings& settings); + bool StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value& value, TTableSettings& settings, ETableType tableType, bool alter = false); bool StoreDataSourceSettingsEntry(const TIdentifier& id, const TRule_table_setting_value* value, std::map<TString, TDeferredAtom>& result); - bool ResetTableSettingsEntry(const TIdentifier& id, TTableSettings& settings); + bool ResetTableSettingsEntry(const TIdentifier& id, TTableSettings& settings, ETableType tableType); TNodePtr TypeSimple(const TRule_type_name_simple& node, bool onlyDataAllowed); TNodePtr TypeDecimal(const TRule_type_name_decimal& node); @@ -853,7 +855,9 @@ protected: bool ParseObjectFeatures(std::map<TString, TDeferredAtom> & result, const TRule_object_features & features); bool ParseExternalDataSourceSettings(std::map<TString, TDeferredAtom> & result, const TRule_with_table_settings & settings); bool RoleNameClause(const TRule_role_name& node, TDeferredAtom& result, bool allowSystemRoles); - bool RoleParameters(const TRule_create_user_option& node, TRoleParameters& result) ; + bool RoleParameters(const TRule_create_user_option& node, TRoleParameters& result); + + bool ValidateExternalTable(const TCreateTableParameters& params); private: bool SimpleTableRefCoreImpl(const TRule_simple_table_ref_core& node, TTableRef& result); static bool IsValidFrameSettings(TContext& ctx, const TFrameSpecification& frameSpec, size_t sortSpecSize); @@ -2256,10 +2260,47 @@ namespace { } bool TSqlTranslation::StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value* value, + TTableSettings& settings, ETableType tableType, bool alter, bool reset) { + switch (tableType) { + case ETableType::ExternalTable: + return StoreExternalTableSettingsEntry(id, value, settings); + case ETableType::Table: + case ETableType::TableStore: + return StoreTableSettingsEntry(id, value, settings, alter, reset); + } +} + +bool TSqlTranslation::StoreExternalTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value* value, TTableSettings& settings) { + if (to_lower(id.Name) == "data_source") { + TDeferredAtom dataSource; + if (!StoreString(*value, dataSource, Ctx, to_upper(id.Name))) { + return false; + } + TString service = Context().Scoped->CurrService; + TDeferredAtom cluster = Context().Scoped->CurrCluster; + TNodePtr root = new TAstListNodeImpl(Ctx.Pos()); + root->Add("String", Ctx.GetPrefixedPath(service, cluster, dataSource)); + settings.DataSourcePath = root; + } else if (to_lower(id.Name) == "location") { + if (!StoreString(*value, settings.Location, Ctx)) { + Ctx.Error() << to_upper(id.Name) << " value should be a string literal"; + return false; + } + } else { + settings.ExternalSourceParameters.emplace_back(id, nullptr); + auto& parameter = settings.ExternalSourceParameters.back(); + if (!StoreString(*value, parameter.second, Ctx)) { + Ctx.Error() << to_upper(id.Name) << " value should be a string literal"; + return false; + } + } + return true; +} + +bool TSqlTranslation::StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value* value, TTableSettings& settings, bool alter, bool reset) { YQL_ENSURE(value || reset); YQL_ENSURE(!reset || reset & alter); - if (to_lower(id.Name) == "compaction_policy") { if (reset) { Ctx.Error() << to_upper(id.Name) << " reset is not supported"; @@ -2399,23 +2440,23 @@ bool TSqlTranslation::StoreTableSettingsEntry(const TIdentifier& id, const TRule } bool TSqlTranslation::StoreTableSettingsEntry(const TIdentifier& id, const TRule_table_setting_value& value, - TTableSettings& settings, bool alter) { - return StoreTableSettingsEntry(id, &value, settings, alter, false); + TTableSettings& settings, ETableType tableType, bool alter) { + return StoreTableSettingsEntry(id, &value, settings, tableType, alter, false); } -bool TSqlTranslation::ResetTableSettingsEntry(const TIdentifier& id, TTableSettings& settings) { - return StoreTableSettingsEntry(id, nullptr, settings, true, true); +bool TSqlTranslation::ResetTableSettingsEntry(const TIdentifier& id, TTableSettings& settings, ETableType tableType) { + return StoreTableSettingsEntry(id, nullptr, settings, tableType, true, true); } -bool TSqlTranslation::CreateTableSettings(const TRule_with_table_settings& settingsNode, TTableSettings& settings) { +bool TSqlTranslation::CreateTableSettings(const TRule_with_table_settings& settingsNode, TCreateTableParameters& params) { const auto& firstEntry = settingsNode.GetRule_table_settings_entry3(); if (!StoreTableSettingsEntry(IdEx(firstEntry.GetRule_an_id1(), *this), firstEntry.GetRule_table_setting_value3(), - settings)) { + params.TableSettings, params.TableType)) { return false; } for (auto& block : settingsNode.GetBlock4()) { const auto& entry = block.GetRule_table_settings_entry2(); - if (!StoreTableSettingsEntry(IdEx(entry.GetRule_an_id1(), *this), entry.GetRule_table_setting_value3(), settings)) { + if (!StoreTableSettingsEntry(IdEx(entry.GetRule_an_id1(), *this), entry.GetRule_table_setting_value3(), params.TableSettings, params.TableType)) { return false; } } @@ -8799,18 +8840,20 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& case TRule_sql_stmt_core::kAltSqlStmtCore4: { Ctx.BodyPart(); const auto& rule = core.GetAlt_sql_stmt_core4().GetRule_create_table_stmt1(); - const bool isTablestore = rule.GetToken2().GetId() == SQLv1LexerTokens::TOKEN_TABLESTORE; + const auto& block = rule.GetBlock2(); + ETableType tableType = ETableType::Table; + if (block.HasAlt2() && block.GetAlt2().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_TABLESTORE) { + tableType = ETableType::TableStore; + } else if (block.HasAlt3() && block.GetAlt3().GetToken1().GetId() == SQLv1LexerTokens::TOKEN_EXTERNAL) { + tableType = ETableType::ExternalTable; + } TTableRef tr; if (!SimpleTableRefImpl(rule.GetRule_simple_table_ref3(), tr)) { return false; } - TCreateTableParameters params; - if (isTablestore) { - params.TableType = ETableType::TableStore; - } - + TCreateTableParameters params{.TableType=tableType}; if (!CreateTableEntry(rule.GetRule_create_table_entry5(), params)) { return false; } @@ -8827,7 +8870,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& } if (rule.HasBlock9()) { - if (isTablestore) { + if (tableType == ETableType::TableStore) { Context().Error(GetPos(rule.GetBlock9().GetRule_table_partition_by1().GetToken1())) << "PARTITION BY is not supported for TABLESTORE"; return false; @@ -8840,7 +8883,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& } if (rule.HasBlock10()) { - if (!CreateTableSettings(rule.GetBlock10().GetRule_with_table_settings1(), params.TableSettings)) { + if (!CreateTableSettings(rule.GetBlock10().GetRule_with_table_settings1(), params)) { return false; } } @@ -8851,13 +8894,25 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& return false; } + if (!ValidateExternalTable(params)) { + return false; + } + AddStatementToBlocks(blocks, BuildCreateTable(Ctx.Pos(), tr, params, Ctx.Scoped)); break; } case TRule_sql_stmt_core::kAltSqlStmtCore5: { Ctx.BodyPart(); const auto& rule = core.GetAlt_sql_stmt_core5().GetRule_drop_table_stmt1(); - const bool isTablestore = rule.GetToken2().GetId() == SQLv1LexerTokens::TOKEN_TABLESTORE; + const auto& block = rule.GetBlock2(); + ETableType tableType = ETableType::Table; + if (block.HasAlt2()) { + tableType = ETableType::TableStore; + } + if (block.HasAlt3()) { + tableType = ETableType::ExternalTable; + } + if (rule.HasBlock3()) { Context().Error(GetPos(rule.GetToken1())) << "IF EXISTS in " << humanStatementName << " is not supported."; @@ -8868,7 +8923,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core& return false; } - AddStatementToBlocks(blocks, BuildDropTable(Ctx.Pos(), tr, isTablestore, Ctx.Scoped)); + AddStatementToBlocks(blocks, BuildDropTable(Ctx.Pos(), tr, tableType, Ctx.Scoped)); break; } case TRule_sql_stmt_core::kAltSqlStmtCore6: { @@ -9661,7 +9716,7 @@ bool TSqlQuery::AlterTableSetTableSetting(const TRule_alter_table_set_table_sett TAlterTableParameters& params) { if (!StoreTableSettingsEntry(IdEx(node.GetRule_an_id2(), *this), node.GetRule_table_setting_value3(), - params.TableSettings, true)) { + params.TableSettings, params.TableType, true)) { return false; } return true; @@ -9672,13 +9727,13 @@ bool TSqlQuery::AlterTableSetTableSetting(const TRule_alter_table_set_table_sett { const auto& firstEntry = node.GetRule_alter_table_setting_entry3(); if (!StoreTableSettingsEntry(IdEx(firstEntry.GetRule_an_id1(), *this), firstEntry.GetRule_table_setting_value3(), - params.TableSettings, true)) { + params.TableSettings, params.TableType, true)) { return false; } for (auto& block : node.GetBlock4()) { const auto& entry = block.GetRule_alter_table_setting_entry2(); if (!StoreTableSettingsEntry(IdEx(entry.GetRule_an_id1(), *this), entry.GetRule_table_setting_value3(), - params.TableSettings, true)) { + params.TableSettings, params.TableType, true)) { return false; } } @@ -9689,12 +9744,12 @@ bool TSqlQuery::AlterTableResetTableSetting(const TRule_alter_table_reset_table_ TAlterTableParameters& params) { const auto& firstEntry = node.GetRule_an_id3(); - if (!ResetTableSettingsEntry(IdEx(firstEntry, *this), params.TableSettings)) { + if (!ResetTableSettingsEntry(IdEx(firstEntry, *this), params.TableSettings, params.TableType)) { return false; } for (auto& block : node.GetBlock4()) { const auto& entry = block.GetRule_an_id2(); - if (!ResetTableSettingsEntry(IdEx(entry, *this), params.TableSettings)) { + if (!ResetTableSettingsEntry(IdEx(entry, *this), params.TableSettings, params.TableType)) { return false; } } @@ -10974,4 +11029,27 @@ bool TSqlTranslation::ParseExternalDataSourceSettings(std::map<TString, TDeferre return true; } +bool TSqlTranslation::ValidateExternalTable(const TCreateTableParameters& params) { + if (params.TableType != ETableType::ExternalTable) { + return true; + } + + if (!params.TableSettings.DataSourcePath) { + Ctx.Error() << "DATA_SOURCE requires key"; + return false; + } + + if (!params.TableSettings.Location) { + Ctx.Error() << "LOCATION requires key"; + return false; + } + + if (params.PkColumns) { + Ctx.Error() << "PRIMARY KEY is not supported for external table"; + return false; + } + + return true; +} + } // namespace NSQLTranslationV1 diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp index a9bf77cb2bd..ff660fbd024 100644 --- a/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/ydb/library/yql/sql/v1/sql_ut.cpp @@ -4706,7 +4706,9 @@ Y_UNIT_TEST_SUITE(ExternalDeclares) { UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:15: Error: Selecting data from monitoring source is not supported\n"); } +} +Y_UNIT_TEST_SUITE(ExternalDataSource) { Y_UNIT_TEST(CreateExternalDataSource) { NYql::TAstParseResult res = SqlToYql(R"( USE plato; @@ -4916,3 +4918,167 @@ Y_UNIT_TEST_SUITE(ExternalDeclares) { UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); } } + +Y_UNIT_TEST_SUITE(ExternalTable) { + Y_UNIT_TEST(CreateExternalTable) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + CREATE EXTERNAL TABLE mytable ( + a int + ) WITH ( + DATA_SOURCE="/Root/mydatasource", + LOCATION="/folder1/*" + ); + )"); + UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('data_source_path (String '"/Root/mydatasource")) '('location (String '"/folder1/*")))) '('tableType 'externalTable)))))#"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tablescheme")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateExternalTableWithTablePrefix) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + pragma TablePathPrefix='/aba'; + CREATE EXTERNAL TABLE mytable ( + a int + ) WITH ( + DATA_SOURCE="mydatasource", + LOCATION="/folder1/*" + ); + )"); + UNIT_ASSERT_C(res.Root, res.Issues.ToOneLineString()); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, "/aba/mydatasource"); + UNIT_ASSERT_STRING_CONTAINS(line, "/aba/mytable"); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tablescheme")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0} }; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(CreateExternalTableObjectStorage) { + auto res = SqlToYql(R"( + USE plato; + CREATE EXTERNAL TABLE mytable ( + a int, + year Int + ) WITH ( + DATA_SOURCE="/Root/mydatasource", + LOCATION="/folder1/*", + FORMAT="json_as_string", + `projection.enabled`="true", + `projection.year.type`="integer", + `projection.year.min`="2010", + `projection.year.max`="2022", + `projection.year.interval`="1", + `projection.month.type`="integer", + `projection.month.min`="1", + `projection.month.max`="12", + `projection.month.interval`="1", + `projection.month.digits`="2", + `storage.location.template`="${year}/${month}", + PARTITONED_BY = "[year, month]" + ); + )"); + UNIT_ASSERT_C(res.IsOk(), res.Issues.ToString()); + } + + Y_UNIT_TEST(CreateExternalTableWithBadArguments) { + ExpectFailWithError(R"( + USE plato; + CREATE EXTERNAL TABLE mytable; + )" , "<main>:3:45: Error: Unexpected token ';' : syntax error...\n\n"); + + ExpectFailWithError(R"( + USE plato; + CREATE EXTERNAL TABLE mytable ( + a int + ); + )" , "<main>:4:23: Error: DATA_SOURCE requires key\n"); + + ExpectFailWithError(R"( + USE plato; + CREATE EXTERNAL TABLE mytable ( + a int + ) WITH ( + DATA_SOURCE="/Root/mydatasource" + ); + )" , "<main>:6:33: Error: LOCATION requires key\n"); + + ExpectFailWithError(R"( + USE plato; + CREATE EXTERNAL TABLE mytable ( + a int + ) WITH ( + LOCATION="/folder1/*" + ); + )" , "<main>:6:30: Error: DATA_SOURCE requires key\n"); + + ExpectFailWithError(R"( + USE plato; + CREATE EXTERNAL TABLE mytable ( + a int, + PRIMARY KEY(a) + ) WITH ( + DATA_SOURCE="/Root/mydatasource", + LOCATION="/folder1/*" + ); + )" , "<main>:8:30: Error: PRIMARY KEY is not supported for external table\n"); + } + + Y_UNIT_TEST(DropExternalTable) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + DROP EXTERNAL TABLE MyExternalTable; + )"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("tablescheme")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } + + Y_UNIT_TEST(DropExternalTableWithTablePrefix) { + NYql::TAstParseResult res = SqlToYql(R"( + USE plato; + pragma TablePathPrefix='/aba'; + DROP EXTERNAL TABLE MyExternalTable; + )"); + UNIT_ASSERT(res.Root); + + TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { + if (word == "Write") { + UNIT_ASSERT_STRING_CONTAINS(line, "/aba/MyExternalTable"); + UNIT_ASSERT_VALUES_EQUAL(TString::npos, line.find("'tablescheme")); + } + }; + + TWordCountHive elementStat = { {TString("Write"), 0}}; + VerifyProgram(res, elementStat, verifyLine); + + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + } +} |