diff options
author | ssmike <ssmike@ydb.tech> | 2023-09-13 15:53:46 +0300 |
---|---|---|
committer | ssmike <ssmike@ydb.tech> | 2023-09-13 16:41:31 +0300 |
commit | f0306787cc9780b9cb5b2cde2240ba27ff45606a (patch) | |
tree | 0a11ca1355d4be9508034db142b2f68fe9db1f3f | |
parent | 056c12b1a2325cf819372866fc1da9025f94efad (diff) | |
download | ydb-f0306787cc9780b9cb5b2cde2240ba27ff45606a.tar.gz |
Implement index auto-choosing
23 files changed, 263 insertions, 73 deletions
diff --git a/ydb/core/kqp/common/kqp_yql.cpp b/ydb/core/kqp/common/kqp_yql.cpp index 27443b4323..b7cf36d3a8 100644 --- a/ydb/core/kqp/common/kqp_yql.cpp +++ b/ydb/core/kqp/common/kqp_yql.cpp @@ -163,6 +163,9 @@ TKqpReadTableSettings ParseInternal(const TCoNameValueTupleList& node) { } else if (name == TKqpReadTableSettings::SequentialSettingName) { YQL_ENSURE(tuple.Ref().ChildrenSize() == 2); settings.SequentialInFlight = FromString<ui64>(tuple.Value().Cast<TCoAtom>().Value()); + } else if (name == TKqpReadTableSettings::ForcePrimaryName) { + YQL_ENSURE(tuple.Ref().ChildrenSize() == 1); + settings.ForcePrimary = true; } else { YQL_ENSURE(false, "Unknown KqpReadTable setting name '" << name << "'"); } @@ -223,6 +226,14 @@ NNodes::TCoNameValueTupleList TKqpReadTableSettings::BuildNode(TExprContext& ctx .Done()); } + if (ForcePrimary) { + settings.emplace_back( + Build<TCoNameValueTuple>(ctx, pos) + .Name() + .Build(ForcePrimaryName) + .Done()); + } + if (Sorted) { settings.emplace_back( Build<TCoNameValueTuple>(ctx, pos) diff --git a/ydb/core/kqp/common/kqp_yql.h b/ydb/core/kqp/common/kqp_yql.h index 08636cbe1d..3549a1a985 100644 --- a/ydb/core/kqp/common/kqp_yql.h +++ b/ydb/core/kqp/common/kqp_yql.h @@ -51,12 +51,14 @@ struct TKqpReadTableSettings { static constexpr TStringBuf ReverseSettingName = "Reverse"; static constexpr TStringBuf SortedSettingName = "Sorted"; static constexpr TStringBuf SequentialSettingName = "Sequential"; + static constexpr TStringBuf ForcePrimaryName = "ForcePrimary"; TVector<TString> SkipNullKeys; TExprNode::TPtr ItemsLimit; bool Reverse = false; bool Sorted = false; TMaybe<ui64> SequentialInFlight; + bool ForcePrimary = false; void AddSkipNullKey(const TString& key); void SetItemsLimit(const TExprNode::TPtr& expr) { ItemsLimit = expr; } diff --git a/ydb/core/kqp/compile_service/kqp_compile_actor.cpp b/ydb/core/kqp/compile_service/kqp_compile_actor.cpp index 22894aefe6..2ba469bba7 100644 --- a/ydb/core/kqp/compile_service/kqp_compile_actor.cpp +++ b/ydb/core/kqp/compile_service/kqp_compile_actor.cpp @@ -418,6 +418,7 @@ void ApplyServiceConfig(TKikimrConfiguration& kqpConfig, const TTableServiceConf kqpConfig.EnableSequences = serviceConfig.GetEnableSequences(); kqpConfig.BindingsMode = RemapBindingsMode(serviceConfig.GetBindingsMode()); kqpConfig.PredicateExtract20 = serviceConfig.GetPredicateExtract20(); + kqpConfig.EnableIndexAutoChooser = serviceConfig.GetEnableIndexAutoChooser(); } IActor* CreateKqpCompileActor(const TActorId& owner, const TKqpSettings::TConstPtr& kqpSettings, diff --git a/ydb/core/kqp/compile_service/kqp_compile_service.cpp b/ydb/core/kqp/compile_service/kqp_compile_service.cpp index 24761a849a..fac856ee2c 100644 --- a/ydb/core/kqp/compile_service/kqp_compile_service.cpp +++ b/ydb/core/kqp/compile_service/kqp_compile_service.cpp @@ -402,6 +402,8 @@ private: bool defaultSyntaxVersion = Config.GetSqlVersion(); bool enableKqpImmediateEffects = Config.GetEnableKqpImmediateEffects(); + bool indexAutoChooser = Config.GetEnableIndexAutoChooser(); + Config.Swap(event.MutableConfig()->MutableTableServiceConfig()); LOG_INFO(*TlsActivationContext, NKikimrServices::KQP_COMPILE_SERVICE, "Updated config"); @@ -418,7 +420,8 @@ private: Config.GetEnablePredicateExtractForScanQueries() != enableKqpScanQueryPredicateExtract || Config.GetPredicateExtract20() != predicateExtract20 || Config.GetEnableSequentialReads() != enableSequentialReads || - Config.GetEnableKqpImmediateEffects() != enableKqpImmediateEffects) { + Config.GetEnableKqpImmediateEffects() != enableKqpImmediateEffects || + Config.GetEnableIndexAutoChooser() != indexAutoChooser) { LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_COMPILE_SERVICE, "Iterator read flags was changed. StreamLookup from " << enableKqpDataQueryStreamLookup << diff --git a/ydb/core/kqp/opt/kqp_opt_kql.cpp b/ydb/core/kqp/opt/kqp_opt_kql.cpp index 0083d6ba7d..ecee50c446 100644 --- a/ydb/core/kqp/opt/kqp_opt_kql.cpp +++ b/ydb/core/kqp/opt/kqp_opt_kql.cpp @@ -120,20 +120,22 @@ bool HasIndexesToWrite(const TKikimrTableDescription& tableData) { return hasIndexesToWrite; } -TExprBase BuildReadTable(const TCoAtomList& columns, TPositionHandle pos, const TKikimrTableDescription& tableData, +TExprBase BuildReadTable(const TCoAtomList& columns, TPositionHandle pos, const TKikimrTableDescription& tableData, bool forcePrimary, TExprContext& ctx, const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx) { TExprNode::TPtr readTable; const auto& tableMeta = BuildTableMeta(tableData, pos, ctx); + TKqpReadTableSettings settings; + settings.ForcePrimary = forcePrimary; + if (UseReadTableRanges(tableData, kqpCtx)) { readTable = Build<TKqlReadTableRanges>(ctx, pos) .Table(tableMeta) .Ranges<TCoVoid>() .Build() .Columns(columns) - .Settings() - .Build() + .Settings(settings.BuildNode(ctx, pos)) .ExplainPrompt() .Build() .Done().Ptr(); @@ -147,8 +149,7 @@ TExprBase BuildReadTable(const TCoAtomList& columns, TPositionHandle pos, const .Build() .Build() .Columns(columns) - .Settings() - .Build() + .Settings(settings.BuildNode(ctx, pos)) .Done().Ptr(); } @@ -156,12 +157,12 @@ TExprBase BuildReadTable(const TCoAtomList& columns, TPositionHandle pos, const } -TExprBase BuildReadTable(const TKiReadTable& read, const TKikimrTableDescription& tableData, +TExprBase BuildReadTable(const TKiReadTable& read, const TKikimrTableDescription& tableData, bool forcePrimary, bool withSystemColumns, TExprContext& ctx, const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx) { const auto& columns = read.GetSelectColumns(ctx, tableData, withSystemColumns); - auto readNode = BuildReadTable(columns, read.Pos(), tableData, ctx, kqpCtx); + auto readNode = BuildReadTable(columns, read.Pos(), tableData, forcePrimary, ctx, kqpCtx); return readNode; } @@ -407,7 +408,7 @@ TExprBase BuildRowsToDelete(const TKikimrTableDescription& tableData, bool withS const auto tableMeta = BuildTableMeta(tableData, pos, ctx); const auto tableColumns = BuildColumnsList(tableData, pos, ctx, withSystemColumns); - const auto allRows = BuildReadTable(tableColumns, pos, tableData, ctx, kqpCtx); + const auto allRows = BuildReadTable(tableColumns, pos, tableData, false, ctx, kqpCtx); return Build<TCoFilter>(ctx, pos) .Input(allRows) @@ -477,7 +478,7 @@ TVector<TExprBase> BuildDeleteTableWithIndex(const TKiDeleteTable& del, const TK TExprBase BuildRowsToUpdate(const TKikimrTableDescription& tableData, bool withSystemColumns, const TCoLambda& filter, const TPositionHandle pos, TExprContext& ctx, const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx) { - auto kqlReadTable = BuildReadTable(BuildColumnsList(tableData, pos, ctx, withSystemColumns), pos, tableData, ctx, kqpCtx); + auto kqlReadTable = BuildReadTable(BuildColumnsList(tableData, pos, ctx, withSystemColumns), pos, tableData, false, ctx, kqpCtx); return Build<TCoFilter>(ctx, pos) .Input(kqlReadTable) @@ -703,9 +704,10 @@ TExprNode::TPtr HandleReadTable(const TKiReadTable& read, TExprContext& ctx, con YQL_ENSURE(key.Extract(read.TableKey().Ref())); YQL_ENSURE(key.GetKeyType() == TKikimrKey::Type::Table); auto& tableData = GetTableData(tablesData, read.DataSource().Cluster(), key.GetTablePath()); + auto view = key.GetView(); - if (key.GetView()) { - const auto& indexName = key.GetView().GetRef(); + if (view && !view->PrimaryFlag) { + const auto& indexName = view->Name; if (!ValidateTableHasIndex(tableData.Metadata, ctx, read.Pos())) { return nullptr; } @@ -732,7 +734,7 @@ TExprNode::TPtr HandleReadTable(const TKiReadTable& read, TExprContext& ctx, con return BuildReadTableIndex(read, tableData, indexName, withSystemColumns, ctx, kqpCtx).Ptr(); } - return BuildReadTable(read, tableData, withSystemColumns, ctx, kqpCtx).Ptr(); + return BuildReadTable(read, tableData, view && view->PrimaryFlag, withSystemColumns, ctx, kqpCtx).Ptr(); } TExprBase WriteTableSimple(const TKiWriteTable& write, const TCoAtomList& inputColumns, diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_ranges_predext.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log_ranges_predext.cpp index 91cd3f0b84..97f8ab6ec0 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log_ranges_predext.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log_ranges_predext.cpp @@ -227,8 +227,8 @@ TExprBase KqpPushExtractedPredicateToReadTable(TExprBase node, TExprContext& ctx indexName = maybeIndexRead.Cast().Index(); } + auto readSettings = TKqpReadTableSettings::Parse(read.Settings()); const auto& mainTableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, read.Table().Path()); - auto& tableDesc = indexName ? kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, mainTableDesc.Metadata->GetIndexMetadata(TString(indexName.Cast())).first->Name) : mainTableDesc; THashSet<TString> possibleKeys; TPredicateExtractorSettings settings; @@ -238,6 +238,11 @@ TExprBase KqpPushExtractedPredicateToReadTable(TExprBase node, TExprContext& ctx if (!kqpCtx.Config->PredicateExtract20) { // test for trivial cases (explicit literals or parameters) + auto& tableDesc = indexName + ? kqpCtx.Tables->ExistingTable( + kqpCtx.Cluster, + mainTableDesc.Metadata->GetIndexMetadata(TString(indexName.Cast())).first->Name) + : mainTableDesc; if (auto expr = TryBuildTrivialReadTable(flatmap, read, *readMatch, tableDesc, ctx, kqpCtx, indexName)) { return expr.Cast(); } @@ -246,11 +251,40 @@ TExprBase KqpPushExtractedPredicateToReadTable(TExprBase node, TExprContext& ctx } auto extractor = MakePredicateRangeExtractor(settings); - YQL_ENSURE(tableDesc.SchemeNode); + YQL_ENSURE(mainTableDesc.SchemeNode); bool prepareSuccess = extractor->Prepare(flatmap.Lambda().Ptr(), *mainTableDesc.SchemeNode, possibleKeys, ctx, typesCtx); YQL_ENSURE(prepareSuccess); + if (!indexName.IsValid() && !readSettings.ForcePrimary && kqpCtx.Config->EnableIndexAutoChooser) { + TVector<std::tuple<size_t, size_t, TMaybe<TString>>> indices; + { + auto buildResult = extractor->BuildComputeNode(mainTableDesc.Metadata->KeyColumnNames, ctx, typesCtx); + indices.push_back(std::make_tuple( + buildResult.UsedPrefixLen, + buildResult.PointPrefixLen, + TMaybe<TString>())); + } + for (auto& index : mainTableDesc.Metadata->Indexes) { + if (index.Type == TIndexDescription::EType::GlobalSync) { + auto buildResult = extractor->BuildComputeNode(index.KeyColumns, ctx, typesCtx); + indices.push_back(std::make_tuple( + buildResult.UsedPrefixLen, + buildResult.PointPrefixLen, + TMaybe<TString>(index.Name))); + } + } + if (!indices.empty()) { + auto it = std::max_element(indices.begin(), indices.end()); + auto name = std::get<TMaybe<TString>>(*it); + if (name) { + indexName = ctx.NewAtom(read.Pos(), *name); + } + } + } + + auto& tableDesc = indexName ? kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, mainTableDesc.Metadata->GetIndexMetadata(TString(indexName.Cast())).first->Name) : mainTableDesc; + auto buildResult = extractor->BuildComputeNode(tableDesc.Metadata->KeyColumnNames, ctx, typesCtx); TExprNode::TPtr ranges = buildResult.ComputeNode; diff --git a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp index 3ac2ec2431..8707c494cb 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp @@ -265,7 +265,7 @@ public: case NKikimrSchemeOp::TAuth::kNone: properties["authMethod"] = "SERVICE_ACCOUNT"; break; - + case NKikimrSchemeOp::TAuth::kBasic: properties["authMethod"] = "BASIC"; properties["login"] = metadata.ExternalSource.DataSourceAuth.GetBasic().GetLogin(); diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp index 37afaad197..017049d602 100644 --- a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp @@ -123,8 +123,9 @@ struct TKiExploreTxResults { YQL_ENSURE(tableMeta, "Empty table metadata"); bool uncommittedChangesRead = false; - if (key.GetView()) { - const auto& indexName = key.GetView().GetRef(); + auto view = key.GetView(); + if (view && view->Name) { + const auto& indexName = view->Name; const auto indexTablePath = IKikimrGateway::CreateIndexTablePath(tableMeta->Name, indexName); auto indexIt = std::find_if(tableMeta->Indexes.begin(), tableMeta->Indexes.end(), [&indexName](const auto& index){ diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.cpp b/ydb/core/kqp/provider/yql_kikimr_provider.cpp index 568c673f60..ffaa3f4594 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_provider.cpp @@ -407,6 +407,10 @@ bool TKikimrKey::Extract(const TExprNode& key) { auto tag = key.Child(i)->Child(0); if (tag->Content() == TStringBuf("view")) { const TExprNode* viewNode = key.Child(i)->Child(1); + if (viewNode->ChildrenSize() == 0 && viewNode->IsList()) { + View = {"", true}; + continue; + } if (!viewNode->IsCallable("String")) { Ctx.AddError(TIssue(Ctx.GetPosition(viewNode->Pos()), "Expected String")); return false; @@ -420,8 +424,20 @@ bool TKikimrKey::Extract(const TExprNode& key) { Ctx.AddError(TIssue(Ctx.GetPosition(viewNode->Child(0)->Pos()), "Secondary index name must not be empty")); return false; } - View = viewNode->Child(0)->Content(); + if (View) { + Ctx.AddError(TIssue(Ctx.GetPosition(tag->Pos()), "Incosistent view tags")); + return false; + } + + View = TViewDescription{TString(viewNode->Child(0)->Content())}; + } else if (tag->Content() == TStringBuf("primary_view")) { + if (View) { + Ctx.AddError(TIssue(Ctx.GetPosition(tag->Pos()), "Incosistent view tags")); + return false; + } + + View = TViewDescription{"", true}; } else { Ctx.AddError(TIssue(Ctx.GetPosition(tag->Pos()), TStringBuilder() << "Unexpected tag for kikimr key child: " << tag->Content())); return false; diff --git a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h index ac9b8a7fe7..9e9ac698a9 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h +++ b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h @@ -76,6 +76,11 @@ public: Permission }; + struct TViewDescription { + TString Name; + bool PrimaryFlag = false; + }; + public: TKikimrKey(TExprContext& ctx) : Ctx(ctx) {} @@ -115,7 +120,7 @@ public: return Target; } - const TMaybe<TString>& GetView() const { + const TMaybe<TViewDescription>& GetView() const { return View; } @@ -139,7 +144,7 @@ private: TMaybe<Type> KeyType; TString Target; TMaybe<TString> ObjectType; - TMaybe<TString> View; + TMaybe<TViewDescription> View; }; struct TKiDataQueryBlockSettings { diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.h b/ydb/core/kqp/provider/yql_kikimr_settings.h index 03df80047f..141a0307c6 100644 --- a/ydb/core/kqp/provider/yql_kikimr_settings.h +++ b/ydb/core/kqp/provider/yql_kikimr_settings.h @@ -152,6 +152,7 @@ struct TKikimrConfiguration : public TKikimrSettings, public NCommon::TSettingDi bool EnablePreparedDdl = false; bool EnableSequences = false; NSQLTranslation::EBindingsMode BindingsMode = NSQLTranslation::EBindingsMode::ENABLED; + bool EnableIndexAutoChooser = false; }; } diff --git a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp index f24617a2e4..afc17f7327 100644 --- a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp @@ -140,13 +140,14 @@ private: return TStatus::Error; } - if (const auto& view = key.GetView()) { + const auto& view = key.GetView(); + if (view && view->Name) { if (!ValidateTableHasIndex(tableDesc->Metadata, ctx, node.Pos())) { return TStatus::Error; } - if (tableDesc->Metadata->GetIndexMetadata(view.GetRef()).first == nullptr) { + if (tableDesc->Metadata->GetIndexMetadata(view->Name).first == nullptr) { ctx.AddError(YqlIssue(ctx.GetPosition(node.Pos()), TIssuesIds::KIKIMR_SCHEME_ERROR, TStringBuilder() - << "Required global index not found, index name: " << view.GetRef())); + << "Required global index not found, index name: " << view->Name)); return TStatus::Error; } } @@ -425,7 +426,7 @@ private: const auto& columnInfo = table->Metadata->Columns.at(keyColumnName); if (rowType->FindItem(keyColumnName)) { continue; - } + } if (!columnInfo.IsDefaultKindDefined()) { ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder() << "Missing key column in input: " << keyColumnName @@ -846,7 +847,7 @@ virtual TStatus HandleCreateTable(TKiCreateTable create, TExprContext& ctx) over const TTypeAnnotationNode* columnAnnotation = columnInfo->second; if (columnAnnotation->HasOptional()) { - columnAnnotation = columnAnnotation->Cast<TOptionalExprType>()->GetItemType(); + columnAnnotation = columnAnnotation->Cast<TOptionalExprType>()->GetItemType(); } if (!IsSameAnnotation(*type, *columnAnnotation)) { diff --git a/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp b/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp index 6839ee8e6b..1fe0006bca 100644 --- a/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp +++ b/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp @@ -3871,6 +3871,47 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { CompareYson(R"([[["id"]]])", FormatResultSetYson(result.GetResultSet(0))); } + + Y_UNIT_TEST(PrimaryView) { + auto kikimr = DefaultKikimrRunner(); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + CreateSampleTablesWithIndex(session); + + NYdb::NTable::TExecDataQuerySettings querySettings; + querySettings.CollectQueryStats(ECollectQueryStatsMode::Profile); + + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + SELECT * FROM `/Root/SecondaryKeys` VIEW @primary WHERE Fk <= 1; + )", TTxControl::BeginTx(TTxSettings::SerializableRW()), querySettings).GetValueSync(); + AssertSuccessResult(result); + AssertTableReads(result, "/Root/SecondaryKeys/Index/indexImplTable", 0); + } + + Y_UNIT_TEST(AutoChooseIndex) { + TKikimrSettings settings; + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableIndexAutoChooser(true); + settings.SetAppConfig(appConfig); + + TKikimrRunner kikimr(settings); + + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + CreateSampleTablesWithIndex(session); + + NYdb::NTable::TExecDataQuerySettings querySettings; + querySettings.CollectQueryStats(ECollectQueryStatsMode::Profile); + + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + SELECT * FROM `/Root/SecondaryKeys` WHERE Fk <= 1; + )", TTxControl::BeginTx(TTxSettings::SerializableRW()), querySettings).GetValueSync(); + AssertSuccessResult(result); + AssertTableReads(result, "/Root/SecondaryKeys/Index/indexImplTable", 1); + } + } } // namespace NKikimr::NKqp diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 11bbed3113..7513e7be55 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -1381,6 +1381,7 @@ message TTableServiceConfig { } optional EChannelTransportVersion ChannelTransportVersion = 46 [default = CTV_OOB_PICKLE_1_0]; + optional bool EnableIndexAutoChooser = 49 [default = false]; }; message TQueryServiceConfig { diff --git a/ydb/library/yql/sql/v1/SQLv1.g.in b/ydb/library/yql/sql/v1/SQLv1.g.in index 7dca784006..4cce86125a 100644 --- a/ydb/library/yql/sql/v1/SQLv1.g.in +++ b/ydb/library/yql/sql/v1/SQLv1.g.in @@ -771,10 +771,10 @@ define_action_or_subquery_body: SEMICOLON* (sql_stmt_core (SEMICOLON+ sql_stmt_c if_stmt: EVALUATE? IF expr do_stmt (ELSE do_stmt)?; for_stmt: EVALUATE? FOR bind_parameter IN expr do_stmt (ELSE do_stmt)?; -table_ref: (cluster_expr DOT)? AT? (table_key | an_id_expr LPAREN (table_arg (COMMA table_arg)* COMMA?)? RPAREN | bind_parameter (LPAREN expr_list? RPAREN)? (VIEW an_id)?) table_hints?; +table_ref: (cluster_expr DOT)? AT? (table_key | an_id_expr LPAREN (table_arg (COMMA table_arg)* COMMA?)? RPAREN | bind_parameter (LPAREN expr_list? RPAREN)? (VIEW view_name)?) table_hints?; -table_key: id_table_or_type (VIEW an_id)?; -table_arg: AT? named_expr (VIEW an_id)?; +table_key: id_table_or_type (VIEW view_name)?; +table_arg: AT? named_expr (VIEW view_name)?; table_hints: WITH (table_hint | LPAREN table_hint (COMMA table_hint)* RPAREN); table_hint: an_id_hint (EQUALS (type_name_tag | LPAREN type_name_tag (COMMA type_name_tag)* COMMA? RPAREN))? @@ -1013,6 +1013,8 @@ an_id_hint: id_hint | STRING_VALUE; an_id_pure: identifier | STRING_VALUE; an_id_as_compat: id_as_compat | STRING_VALUE; +view_name: an_id | AT PRIMARY; + opt_id_prefix: (an_id DOT)?; cluster_expr: (an_id COLON)? (pure_column_or_named | ASTERISK); diff --git a/ydb/library/yql/sql/v1/node.cpp b/ydb/library/yql/sql/v1/node.cpp index 6d30721972..f7af15ec8d 100644 --- a/ydb/library/yql/sql/v1/node.cpp +++ b/ydb/library/yql/sql/v1/node.cpp @@ -274,6 +274,12 @@ bool INode::SetViewName(TContext& ctx, TPosition pos, const TString& view) { return false; } +bool INode::SetPrimaryView(TContext& ctx, TPosition pos) { + Y_UNUSED(pos); + ctx.Error() << "Node not support primary views"; + return false; +} + void INode::UseAsInner() { AsInner = true; } @@ -879,6 +885,17 @@ TNodePtr BuildQuotedAtom(TPosition pos, const TString& content, ui32 flags) { return new TQuotedAtomNode(pos, content, flags); } + +TNodePtr ITableKeys::AddView(TNodePtr key, const TViewDescription& view) { + if (view.PrimaryFlag) { + return L(key, Q(Y(Q("primary_view")))); + } else if (!view.empty()) { + return L(key, Q(Y(Q("view"), Y("String", BuildQuotedAtom(Pos, view.ViewName))))); + } else { + return key; + } +} + TString TColumns::AddUnnamed() { TString desiredResult = TStringBuilder() << "column" << List.size(); if (!All) { @@ -2131,7 +2148,7 @@ public: const TVector<TIdPart>& GetParts() const { return Ids; } - + protected: void DoUpdateState() const override { YQL_ENSURE(Node); @@ -2503,7 +2520,7 @@ bool TUdfNode::DoInit(TContext& ctx, ISource* src) { } TTupleNode* as_tuple = dynamic_cast<TTupleNode*>(Args[0].Get()); - + if (!as_tuple || as_tuple->GetTupleSize() < 1) { ctx.Error(Pos) << "Udf: first argument must be a callable, like Foo::Bar"; return false; @@ -2520,7 +2537,7 @@ bool TUdfNode::DoInit(TContext& ctx, ISource* src) { ModuleName = function->ModuleName(); TVector<TNodePtr> external; external.reserve(as_tuple->GetTupleSize() - 1); - + for (size_t i = 1; i < as_tuple->GetTupleSize(); ++i) { // TODO(): support named args in GetFunctionArgColumnStatus TNodePtr current = as_tuple->GetTupleElement(i); @@ -2532,7 +2549,7 @@ bool TUdfNode::DoInit(TContext& ctx, ISource* src) { } ExternalTypesTuple = new TCallNodeImpl(Pos, "TupleType", external); - + if (Args.size() == 1) { return true; } @@ -2775,7 +2792,7 @@ TSourcePtr TryMakeSourceFromExpression(TContext& ctx, const TString& currService } if (auto literal = node->GetLiteral("String")) { - TNodePtr tableKey = BuildTableKey(node->GetPos(), currService, currCluster, TDeferredAtom(node->GetPos(), *literal), view); + TNodePtr tableKey = BuildTableKey(node->GetPos(), currService, currCluster, TDeferredAtom(node->GetPos(), *literal), {view}); TTableRef table(ctx.MakeName("table"), currService, currCluster, tableKey); table.Options = BuildInputOptions(node->GetPos(), GetContextHints(ctx)); return BuildTableSource(node->GetPos(), table); @@ -2787,7 +2804,7 @@ TSourcePtr TryMakeSourceFromExpression(TContext& ctx, const TString& currService } auto wrappedNode = node->Y("EvaluateAtom", node); - TNodePtr tableKey = BuildTableKey(node->GetPos(), currService, currCluster, TDeferredAtom(wrappedNode, ctx), view); + TNodePtr tableKey = BuildTableKey(node->GetPos(), currService, currCluster, TDeferredAtom(wrappedNode, ctx), {view}); TTableRef table(ctx.MakeName("table"), currService, currCluster, tableKey); table.Options = BuildInputOptions(node->GetPos(), GetContextHints(ctx)); return BuildTableSource(node->GetPos(), table); diff --git a/ydb/library/yql/sql/v1/node.h b/ydb/library/yql/sql/v1/node.h index 428a7f9de6..7dd8590d62 100644 --- a/ydb/library/yql/sql/v1/node.h +++ b/ydb/library/yql/sql/v1/node.h @@ -171,6 +171,7 @@ namespace NSQLTranslationV1 { virtual void CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs); virtual TPtr WindowSpecFunc(const TPtr& type) const; virtual bool SetViewName(TContext& ctx, TPosition pos, const TString& view); + virtual bool SetPrimaryView(TContext& ctx, TPosition pos); void UseAsInner(); virtual bool UsedSubquery() const; virtual bool IsSelect() const; @@ -465,6 +466,14 @@ namespace NSQLTranslationV1 { TWinRank(TPosition pos, const TString& opName, i32 minArgs, i32 maxArgs, const TVector<TNodePtr>& args); }; + struct TViewDescription { + TString ViewName = ""; + bool PrimaryFlag = false; + + bool empty() const { return *this == TViewDescription(); } + bool operator == (const TViewDescription&) const = default; + }; + class ITableKeys: public INode { public: enum class EBuildKeysMode { @@ -478,6 +487,9 @@ namespace NSQLTranslationV1 { virtual const TString* GetTableName() const; virtual TNodePtr BuildKeys(TContext& ctx, EBuildKeysMode mode) = 0; + protected: + TNodePtr AddView(TNodePtr key, const TViewDescription& view); + private: /// all TableKeys no clonnable TPtr DoClone() const final { @@ -900,7 +912,7 @@ namespace NSQLTranslationV1 { bool HasAt = false; TNodePtr Expr; TDeferredAtom Id; - TString View; + TViewDescription View; }; class TTableRows final : public INode { diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp index 809154d8e7..c37489868b 100644 --- a/ydb/library/yql/sql/v1/query.cpp +++ b/ydb/library/yql/sql/v1/query.cpp @@ -15,10 +15,18 @@ using namespace NYql; namespace NSQLTranslationV1 { +bool ValidateView(TPosition pos, TContext& ctx, TStringBuf service, TViewDescription& view) { + if (view.PrimaryFlag && !(service == KikimrProviderName || service == YdbProviderName)) { + ctx.Error(pos) << "@primary is not supported for " << service << " tables"; + return false; + } + return true; +} + class TUniqueTableKey: public ITableKeys { public: TUniqueTableKey(TPosition pos, const TString& service, const TDeferredAtom& cluster, - const TDeferredAtom& name, const TString& view) + const TDeferredAtom& name, const TViewDescription& view) : ITableKeys(pos) , Service(service) , Cluster(cluster) @@ -26,18 +34,25 @@ public: , View(view) , Full(name.GetRepr()) { - if (!View.empty()) { - Full += ":" + View; + if (!View.ViewName.empty()) { + Full += ":" + View.ViewName; } } + bool SetPrimaryView(TContext& ctx, TPosition pos) override { + Y_UNUSED(ctx); + Y_UNUSED(pos); + View = {"", true}; + return true; + } + bool SetViewName(TContext& ctx, TPosition pos, const TString& view) override { Y_UNUSED(ctx); Y_UNUSED(pos); Full = Name.GetRepr(); - View = view; + View = {view}; if (!View.empty()) { - Full = ":" + View; + Full = ":" + View.ViewName; } return true; @@ -48,7 +63,7 @@ public: } TNodePtr BuildKeys(TContext& ctx, ITableKeys::EBuildKeysMode mode) override { - if (View == "@") { + if (View == TViewDescription{"@"}) { auto key = Y("TempTable", Name.Build()); return key; } @@ -63,8 +78,9 @@ public: return nullptr; } auto key = Y("Key", Q(Y(Q(tableScheme ? "tablescheme" : "table"), Y("String", path)))); - if (!View.empty()) { - key = L(key, Q(Y(Q("view"), Y("String", BuildQuotedAtom(Pos, View))))); + key = AddView(key, View); + if (!ValidateView(GetPos(), ctx, Service, View)) { + return nullptr; } if (mode == ITableKeys::EBuildKeysMode::INPUT && IsQueryMode(ctx.Settings.Mode) && @@ -81,12 +97,12 @@ private: TString Service; TDeferredAtom Cluster; TDeferredAtom Name; - TString View; + TViewDescription View; TString Full; }; TNodePtr BuildTableKey(TPosition pos, const TString& service, const TDeferredAtom& cluster, - const TDeferredAtom& name, const TString& view) { + const TDeferredAtom& name, const TViewDescription& view) { return new TUniqueTableKey(pos, service, cluster, name, view); } @@ -242,8 +258,9 @@ public: } key = Y("Key", Q(Y(Q("table"), Y("String", path)))); - if (!arg.View.empty()) { - key = L(key, Q(Y(Q("view"), Y("String", BuildQuotedAtom(Pos, arg.View))))); + key = AddView(key, arg.View); + if (!ValidateView(GetPos(), ctx, Service, arg.View)) { + return nullptr; } } @@ -265,8 +282,9 @@ public: } key = Y("Key", Q(Y(Q("table"), Y("String", path)))); - if (!arg.View.empty()) { - key = L(key, Q(Y(Q("view"), Y("String", BuildQuotedAtom(Pos, arg.View))))); + key = AddView(key, arg.View); + if (!ValidateView(GetPos(), ctx, Service, arg.View)) { + return nullptr; } } @@ -409,8 +427,10 @@ public: auto key = Y("Key", Q(Y(Q("table"), Y("EvaluateExpr", Y("EnsureType", Y("Coalesce", arg.Expr, Y("List", type)), type))))); - if (!arg.View.empty()) { - key = L(key, Q(Y(Q("view"), Y("String", BuildQuotedAtom(Pos, arg.View))))); + + key = AddView(key, arg.View); + if (!ValidateView(GetPos(), ctx, Service, arg.View)) { + return nullptr; } each = L(each, key); } diff --git a/ydb/library/yql/sql/v1/source.h b/ydb/library/yql/sql/v1/source.h index 18b4c62fd7..0c729fc3ef 100644 --- a/ydb/library/yql/sql/v1/source.h +++ b/ydb/library/yql/sql/v1/source.h @@ -289,7 +289,7 @@ namespace NSQLTranslationV1 { TNodePtr BuildDelete(TPosition pos, TScopedStatePtr scoped, const TTableRef& table, TSourcePtr source); // Implemented in query.cpp - TNodePtr BuildTableKey(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TString& view); + TNodePtr BuildTableKey(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& name, const TViewDescription& view); TNodePtr BuildTableKeys(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TString& func, const TVector<TTableArg>& args); TNodePtr BuildTopicKey(TPosition pos, const TDeferredAtom& cluster, const TDeferredAtom& name); TNodePtr BuildInputOptions(TPosition pos, const TTableHints& hints); diff --git a/ydb/library/yql/sql/v1/sql_into_tables.cpp b/ydb/library/yql/sql/v1/sql_into_tables.cpp index 1100576fa4..fefedfadca 100644 --- a/ydb/library/yql/sql/v1/sql_into_tables.cpp +++ b/ydb/library/yql/sql/v1/sql_into_tables.cpp @@ -189,7 +189,7 @@ TNodePtr TSqlIntoTable::Build(const TRule_into_table_stmt& node) { return nullptr; } } else { - table.Keys = BuildTableKey(pos, service, cluster, nameOrAt.second, nameOrAt.first ? "@" : ""); + table.Keys = BuildTableKey(pos, service, cluster, nameOrAt.second, {nameOrAt.first ? "@" : ""}); } Ctx.IncrementMonCounter("sql_insert_clusters", table.Cluster.GetLiteral() ? *table.Cluster.GetLiteral() : "unknown"); diff --git a/ydb/library/yql/sql/v1/sql_translation.cpp b/ydb/library/yql/sql/v1/sql_translation.cpp index 4700fd3b5e..7e7d746fd7 100644 --- a/ydb/library/yql/sql/v1/sql_translation.cpp +++ b/ydb/library/yql/sql/v1/sql_translation.cpp @@ -448,6 +448,17 @@ TString Id(const TRule_an_id_pure& node, TTranslation& ctx) { } } +TViewDescription Id(const TRule_view_name& node, TTranslation& ctx) { + switch (node.Alt_case()) { + case TRule_view_name::kAltViewName1: + return {Id(node.GetAlt_view_name1().GetRule_an_id1(), ctx)}; + case TRule_view_name::kAltViewName2: + return {"", true}; + case TRule_view_name::ALT_NOT_SET: + Y_FAIL("You should change implementation according to grammar changes"); + } +} + bool NamedNodeImpl(const TRule_bind_parameter& node, TString& name, TTranslation& ctx) { // bind_parameter: DOLLAR (an_id_or_type | TRUE | FALSE); TString id; @@ -594,20 +605,20 @@ bool CreateTableIndex(const TRule_table_index& node, TTranslation& ctx, TVector< return true; } -std::pair<TString, TString> TableKeyImpl(const std::pair<bool, TString>& nameWithAt, TString view, TTranslation& ctx) { +std::pair<TString, TViewDescription> TableKeyImpl(const std::pair<bool, TString>& nameWithAt, TViewDescription view, TTranslation& ctx) { if (nameWithAt.first) { - view = "@"; + view = {"@"}; ctx.Context().IncrementMonCounter("sql_features", "AnonymousTable"); } return std::make_pair(nameWithAt.second, view); } -std::pair<TString, TString> TableKeyImpl(const TRule_table_key& node, TTranslation& ctx, bool hasAt) { +std::pair<TString, TViewDescription> TableKeyImpl(const TRule_table_key& node, TTranslation& ctx, bool hasAt) { auto name(Id(node.GetRule_id_table_or_type1(), ctx)); - TString view; + TViewDescription view; if (node.HasBlock2()) { - view = Id(node.GetBlock2().GetRule_an_id2(), ctx); + view = Id(node.GetBlock2().GetRule_view_name2(), ctx); ctx.Context().IncrementMonCounter("sql_features", "View"); } @@ -798,7 +809,7 @@ TMaybe<TTableArg> TSqlTranslation::TableArgImpl(const TRule_table_arg& node) { } if (node.HasBlock3()) { - ret.View = Id(node.GetBlock3().GetRule_an_id2(), *this); + ret.View = Id(node.GetBlock3().GetRule_view_name2(), *this); Context().IncrementMonCounter("sql_features", "View"); } @@ -934,7 +945,7 @@ bool TSqlTranslation::ApplyTableBinding(const TString& binding, TTableRef& tr, T tr.Cluster = TDeferredAtom(Ctx.Pos(), bindingInfo.Cluster); const TString view = ""; - tr.Keys = BuildTableKey(Ctx.Pos(), tr.Service, tr.Cluster, TDeferredAtom(Ctx.Pos(), bindingInfo.Path), view); + tr.Keys = BuildTableKey(Ctx.Pos(), tr.Service, tr.Cluster, TDeferredAtom(Ctx.Pos(), bindingInfo.Path), {view}); return true; } @@ -982,9 +993,9 @@ bool TSqlTranslation::TableRefImpl(const TRule_table_ref& node, TTableRef& resul auto pair = TableKeyImpl(block.GetAlt1().GetRule_table_key1(), *this, hasAt); if (isBinding) { TString binding = pair.first; - TString view = pair.second; - if (!view.empty()) { - YQL_ENSURE(view != "@"); + auto view = pair.second; + if (!view.ViewName.empty()) { + YQL_ENSURE(view != TViewDescription{"@"}); Ctx.Error() << "VIEW is not supported for table bindings"; return false; } @@ -1102,9 +1113,12 @@ bool TSqlTranslation::TableRefImpl(const TRule_table_ref& node, TTableRef& resul TTableHints contextHints = GetContextHints(Ctx); auto ret = BuildInnerSource(Ctx.Pos(), nodePtr, service, cluster); if (alt.HasBlock3()) { - auto view = Id(alt.GetBlock3().GetRule_an_id2(), *this); + auto view = Id(alt.GetBlock3().GetRule_view_name2(), *this); Ctx.IncrementMonCounter("sql_features", "View"); - if (!ret->SetViewName(Ctx, Ctx.Pos(), view)) { + bool result = view.PrimaryFlag + ? ret->SetPrimaryView(Ctx, Ctx.Pos()) + : ret->SetViewName(Ctx, Ctx.Pos(), view.ViewName); + if (!result) { return false; } } @@ -3067,7 +3081,7 @@ bool TSqlTranslation::SimpleTableRefCoreImpl(const TRule_simple_table_ref_core& result = TTableRef(Context().MakeName("table"), service, cluster, nullptr); auto tableOrAt = Id(node.GetAlt_simple_table_ref_core1().GetRule_object_ref1().GetRule_id_or_at2(), *this); - auto tableAndView = TableKeyImpl(tableOrAt, "", *this); + auto tableAndView = TableKeyImpl(tableOrAt, {}, *this); result.Keys = BuildTableKey(Context().Pos(), result.Service, result.Cluster, TDeferredAtom(Context().Pos(), tableAndView.first), tableAndView.second); break; @@ -3091,7 +3105,7 @@ bool TSqlTranslation::SimpleTableRefCoreImpl(const TRule_simple_table_ref_core& TDeferredAtom table; MakeTableFromExpression(Context(), named, table); result = TTableRef(Context().MakeName("table"), service, cluster, nullptr); - result.Keys = BuildTableKey(Context().Pos(), result.Service, result.Cluster, table, at ? "@" : ""); + result.Keys = BuildTableKey(Context().Pos(), result.Service, result.Cluster, table, {at ? "@" : ""}); break; } case TRule_simple_table_ref_core::AltCase::ALT_NOT_SET: diff --git a/ydb/library/yql/sql/v1/sql_translation.h b/ydb/library/yql/sql/v1/sql_translation.h index d9de438fba..a01910b44b 100644 --- a/ydb/library/yql/sql/v1/sql_translation.h +++ b/ydb/library/yql/sql/v1/sql_translation.h @@ -103,9 +103,9 @@ bool PureColumnOrNamedListStr(const TRule_pure_column_or_named_list& node, TTran bool CreateTableIndex(const TRule_table_index& node, TTranslation& ctx, TVector<TIndexDescription>& indexes); -std::pair<TString, TString> TableKeyImpl(const std::pair<bool, TString>& nameWithAt, TString view, TTranslation& ctx); +std::pair<TString, TViewDescription> TableKeyImpl(const std::pair<bool, TString>& nameWithAt, TViewDescription view, TTranslation& ctx); -std::pair<TString, TString> TableKeyImpl(const TRule_table_key& node, TTranslation& ctx, bool hasAt); +std::pair<TString, TViewDescription> TableKeyImpl(const TRule_table_key& node, TTranslation& ctx, bool hasAt); TMaybe<TColumnConstraints> ColumnConstraints(const TRule_column_schema& node, TTranslation& ctx); diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp index db82d61e9e..b51ebb83c5 100644 --- a/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/ydb/library/yql/sql/v1/sql_ut.cpp @@ -1892,7 +1892,7 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, program.find("columnsDefaultValues")); UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, program.find("columnsDefaultValues")); UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, program.find("Write")); - + #if 0 Cerr << program << Endl; #endif @@ -1900,7 +1900,7 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { TWordCountHive elementStat = { {TString("Write"), 0} }; VerifyProgram(res, elementStat); - UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]); } Y_UNIT_TEST(DefaultValueColumn3) { @@ -1946,7 +1946,7 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { if (word == "Write") { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("default")); UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("columnsDefaultValues")); - UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("columnFamilies")); + UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("columnFamilies")); } }; @@ -3156,6 +3156,12 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) { UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:1: Error: Column value is not in source column set\n"); } + Y_UNIT_TEST(PrimaryViewAbortMapReduce) { + NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input VIEW @primary"); + UNIT_ASSERT(!res.Root); + UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: @primary is not supported for yt tables\n"); + } + Y_UNIT_TEST(InsertAbortMapReduce) { NYql::TAstParseResult res = SqlToYql("INSERT OR ABORT INTO plato.Output SELECT key FROM plato.Input"); UNIT_ASSERT(!res.Root); |