diff options
author | Pavel Velikhov <pavelvelikhov@ydb.tech> | 2024-12-13 13:17:55 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-12-13 13:17:55 +0300 |
commit | 19346460a8060a0ed4731edb192745642ff34b3d (patch) | |
tree | 9afc34b501cc641f382d8a182b5692de887c3344 | |
parent | 4b3df28e3fb33f5cb8f0dd3a60bfc8ec6d399dcb (diff) | |
download | ydb-19346460a8060a0ed4731edb192745642ff34b3d.tar.gz |
Eliminate redunant sorting if the inputs are sorted (#11872)
Co-authored-by: Pavel Ivanov <pudge1000-7@ydb.tech>
55 files changed, 1327 insertions, 602 deletions
diff --git a/ydb/core/kqp/opt/kqp_statistics_transformer.cpp b/ydb/core/kqp/opt/kqp_statistics_transformer.cpp index 8db946fd22..f84d747611 100644 --- a/ydb/core/kqp/opt/kqp_statistics_transformer.cpp +++ b/ydb/core/kqp/opt/kqp_statistics_transformer.cpp @@ -58,6 +58,22 @@ void InferStatisticsForReadTable(const TExprNode::TPtr& input, TTypeAnnotationCo new TOptimizerStatistics::TKeyColumns(indexMeta->KeyColumnNames)); } + auto sortedPrefixPtr = TIntrusivePtr<TOptimizerStatistics::TSortColumns>(); + + TVector<TString> sortedPrefixCols; + TVector<TString> sortedPrefixAliases; + + if (inputStats->StorageType == EStorageType::RowStorage && keyColumns) { + for (auto c : keyColumns->Data ) { + sortedPrefixCols.push_back(c); + sortedPrefixAliases.push_back(""); + } + } + + if (sortedPrefixCols.size()) { + sortedPrefixPtr = TIntrusivePtr<TOptimizerStatistics::TSortColumns>(new TOptimizerStatistics::TSortColumns(sortedPrefixCols, sortedPrefixAliases)); + } + /** * We need index statistics to calculate this in the future * Right now we use very small estimates to make sure CBO picks Lookup Joins @@ -82,6 +98,7 @@ void InferStatisticsForReadTable(const TExprNode::TPtr& input, TTypeAnnotationCo keyColumns, inputStats->ColumnStatistics, inputStats->StorageType); + stats->SortColumns = sortedPrefixPtr; YQL_CLOG(TRACE, CoreDq) << "Infer statistics for read table" << stats->ToString(); @@ -100,7 +117,8 @@ void InferStatisticsForKqpTable(const TExprNode::TPtr& input, TTypeAnnotationCon const auto& tableData = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, path.Value()); if (!tableData.Metadata->StatsLoaded && !kqpCtx.Config->OptOverrideStatistics.Get()) { - return; + YQL_CLOG(TRACE, CoreDq) << "Cannot infer statistics for table: " << path.Value(); + //return; } double nRows = tableData.Metadata->RecordsCount; @@ -134,6 +152,24 @@ void InferStatisticsForKqpTable(const TExprNode::TPtr& input, TTypeAnnotationCon } stats->StorageType = storageType; + auto sortedPrefixPtr = TIntrusivePtr<TOptimizerStatistics::TSortColumns>(); + + TVector<TString> sortedPrefixCols; + TVector<TString> sortedPrefixAliases; + + if (stats->StorageType == EStorageType::RowStorage && stats->KeyColumns) { + for (auto c : keyColumns->Data ) { + sortedPrefixCols.push_back(c); + sortedPrefixAliases.push_back(""); + } + } + + if (sortedPrefixCols.size()) { + sortedPrefixPtr = TIntrusivePtr<TOptimizerStatistics::TSortColumns>(new TOptimizerStatistics::TSortColumns(sortedPrefixCols, sortedPrefixAliases)); + } + + stats->SortColumns = sortedPrefixPtr; + YQL_CLOG(TRACE, CoreDq) << "Infer statistics for table: " << path.Value() << ": " << stats->ToString(); typeCtx->SetStats(input.Get(), stats); @@ -153,11 +189,13 @@ void InferStatisticsForSteamLookup(const TExprNode::TPtr& input, TTypeAnnotation auto streamLookup = inputNode.Cast<TKqpCnStreamLookup>(); int nAttrs = streamLookup.Columns().Size(); - auto inputStats = typeCtx->GetStats(streamLookup.Table().Raw()); - if (!inputStats) { + auto tableStats = typeCtx->GetStats(streamLookup.Table().Raw()); + auto inputStats = typeCtx->GetStats(streamLookup.Output().Raw()); + + if (!inputStats || !tableStats) { return; } - auto byteSize = inputStats->ByteSize * (nAttrs / (double) inputStats->Ncols); + auto byteSize = tableStats->ByteSize * (nAttrs / (double) tableStats->Ncols) * inputStats->Selectivity; auto res = std::make_shared<TOptimizerStatistics>( EStatisticsType::BaseTable, @@ -168,6 +206,7 @@ void InferStatisticsForSteamLookup(const TExprNode::TPtr& input, TTypeAnnotation inputStats->KeyColumns, inputStats->ColumnStatistics, inputStats->StorageType); + res->SortColumns = inputStats->SortColumns; typeCtx->SetStats(input.Get(), res); @@ -182,37 +221,15 @@ void InferStatisticsForSteamLookup(const TExprNode::TPtr& input, TTypeAnnotation void InferStatisticsForLookupTable(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx) { auto inputNode = TExprBase(input); auto lookupTable = inputNode.Cast<TKqlLookupTableBase>(); + auto lookupKeys = lookupTable.LookupKeys(); - int nAttrs = lookupTable.Columns().Size(); - double nRows = 0; - double byteSize = 0; - - auto inputStats = typeCtx->GetStats(lookupTable.Table().Raw()); - if (!inputStats) { + auto inputTableStats = typeCtx->GetStats(lookupTable.Table().Raw()); + auto inputLookupStats = typeCtx->GetStats(lookupKeys.Raw()); + if (!inputTableStats || !inputLookupStats) { return; } - if (lookupTable.LookupKeys().Maybe<TCoIterator>()) { - if (inputStats) { - nRows = inputStats->Nrows; - byteSize = inputStats->ByteSize * (nAttrs / (double) inputStats->Ncols); - } else { - return; - } - } else { - nRows = 1; - byteSize = 10; - } - - typeCtx->SetStats(input.Get(), std::make_shared<TOptimizerStatistics>( - EStatisticsType::BaseTable, - nRows, - nAttrs, - byteSize, - 0, - inputStats->KeyColumns, - inputStats->ColumnStatistics, - inputStats->StorageType)); + typeCtx->SetStats(input.Get(), inputLookupStats); } /** @@ -254,13 +271,29 @@ void InferStatisticsForRowsSourceSettings(const TExprNode::TPtr& input, TTypeAnn new TOptimizerStatistics::TKeyColumns(indexMeta->KeyColumnNames)); } + auto sortedPrefixPtr = TIntrusivePtr<TOptimizerStatistics::TSortColumns>(); + + TVector<TString> sortedPrefixCols; + TVector<TString> sortedPrefixAliases; + + if (inputStats->StorageType == EStorageType::RowStorage && keyColumns) { + for (auto c : keyColumns->Data ) { + sortedPrefixCols.push_back(c); + sortedPrefixAliases.push_back(""); + } + } + + if (sortedPrefixCols.size()) { + sortedPrefixPtr = TIntrusivePtr<TOptimizerStatistics::TSortColumns>(new TOptimizerStatistics::TSortColumns(sortedPrefixCols, sortedPrefixAliases)); + } + int nAttrs = sourceSettings.Columns().Size(); double sizePerRow = inputStats->ByteSize / (inputRows==0?1:inputRows); double byteSize = nRows * sizePerRow * (nAttrs / (double)inputStats->Ncols); double cost = inputStats->Cost; - typeCtx->SetStats(input.Get(), std::make_shared<TOptimizerStatistics>( + auto outputStats = std::make_shared<TOptimizerStatistics>( EStatisticsType::BaseTable, nRows, nAttrs, @@ -268,7 +301,12 @@ void InferStatisticsForRowsSourceSettings(const TExprNode::TPtr& input, TTypeAnn cost, keyColumns, inputStats->ColumnStatistics, - inputStats->StorageType)); + inputStats->StorageType); + outputStats->SortColumns = std::move(sortedPrefixPtr); + + YQL_CLOG(TRACE, CoreDq) << "Infer statistics for source settings: " << outputStats->ToString(); + + typeCtx->SetStats(input.Get(), outputStats); } /** @@ -276,10 +314,20 @@ void InferStatisticsForRowsSourceSettings(const TExprNode::TPtr& input, TTypeAnn * Currently we just make up a number for cardinality (5) and set cost to 0 */ void InferStatisticsForIndexLookup(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx) { - typeCtx->SetStats(input.Get(), std::make_shared<TOptimizerStatistics>(EStatisticsType::BaseTable, 5, 5, 20, 0.0)); + auto inputNode = TExprBase(input); + auto lookupIndex = inputNode.Cast<TKqlLookupIndexBase>(); + + auto inputStats = typeCtx->GetStats(lookupIndex.LookupKeys().Raw()); + if (!inputStats) { + return; + } + + typeCtx->SetStats(input.Get(), inputStats); } -void InferStatisticsForReadTableIndexRanges(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx) { +void InferStatisticsForReadTableIndexRanges(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx, + const TKqpOptimizeContext& kqpCtx) { + auto indexRanges = TKqlReadTableIndexRanges(input); auto inputStats = typeCtx->GetStats(indexRanges.Table().Raw()); @@ -292,22 +340,81 @@ void InferStatisticsForReadTableIndexRanges(const TExprNode::TPtr& input, TTypeA indexColumns.push_back(c.StringValue()); } + auto tablePath = indexRanges.Table().Path(); + const auto& tableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, tablePath); + const auto& [indexMeta, _ ] = tableDesc.Metadata->GetIndexMetadata(indexRanges.Index().StringValue()); + + auto sortedColumns = indexMeta->KeyColumnNames; + + TVector<TString> sortedPrefixCols; + TVector<TString> sortedPrefixAliases; + + for (auto c: sortedColumns ) { + sortedPrefixCols.push_back(c); + sortedPrefixAliases.push_back(""); + } + auto indexColumnsPtr = TIntrusivePtr<TOptimizerStatistics::TKeyColumns>(new TOptimizerStatistics::TKeyColumns(indexColumns)); + auto sortedPrefixPtr = TIntrusivePtr<TOptimizerStatistics::TSortColumns>(); + if (sortedPrefixCols.size()) { + sortedPrefixPtr = TIntrusivePtr<TOptimizerStatistics::TSortColumns>(new TOptimizerStatistics::TSortColumns(sortedPrefixCols, sortedPrefixAliases)); + } auto stats = std::make_shared<TOptimizerStatistics>( inputStats->Type, - inputStats->Nrows, + inputStats->Nrows, inputStats->Ncols, inputStats->ByteSize, inputStats->Cost, indexColumnsPtr, inputStats->ColumnStatistics, inputStats->StorageType); + stats->SortColumns = sortedPrefixPtr; typeCtx->SetStats(input.Get(), stats); YQL_CLOG(TRACE, CoreDq) << "Infer statistics for index: " << stats->ToString(); } +void InferStatisticsForLookupJoin(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx) { + auto lookupJoin = TKqlIndexLookupJoinBase(input); + + auto inputStats = typeCtx->GetStats(lookupJoin.Input().Raw()); + if (!inputStats) { + return; + } + + auto sortedPrefix = inputStats->SortColumns; + auto aliasName = lookupJoin.LeftLabel().StringValue(); + + TVector<TString> sortedPrefixCols; + TVector<TString> sortedPrefixAliases; + + if (sortedPrefix) { + sortedPrefixCols = sortedPrefix->Columns; + sortedPrefixAliases = sortedPrefix->Aliases; + if (aliasName != "") { + for (size_t i=0; i<sortedPrefix->Aliases.size(); i++) { + sortedPrefixAliases[i] = aliasName; + } + } + } + + auto sortedPrefixPtr = TIntrusivePtr<TOptimizerStatistics::TSortColumns>(); + if (sortedPrefixCols.size()) { + sortedPrefixPtr = TIntrusivePtr<TOptimizerStatistics::TSortColumns>(new TOptimizerStatistics::TSortColumns(sortedPrefixCols, sortedPrefixAliases)); + } + + auto outputStats = *inputStats; + outputStats.SortColumns = sortedPrefixPtr; + + YQL_CLOG(TRACE, CoreDq) << "Infer statistics for lookup join: " << outputStats.ToString(); + YQL_CLOG(TRACE, CoreDq) << "Added alias: " << aliasName; + + + + typeCtx->SetStats(input.Get(), std::make_shared<TOptimizerStatistics>(std::move(outputStats))); +} + /*** * Infer statistics for result binding of a stage */ @@ -620,7 +727,9 @@ void InferStatisticsForDqSourceWrap(const TExprNode::TPtr& input, TTypeAnnotatio auto rowType = wrapBase.Cast().RowType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>(); if (specific->FullRawRowAvgSize == 0.0) { auto newSpecific = std::make_shared<TS3ProviderStatistics>(*specific); + auto sortColumns = stats->SortColumns; stats = std::make_shared<TOptimizerStatistics>(stats->Type, stats->Nrows, stats->Ncols, stats->ByteSize, stats->Cost, stats->KeyColumns, stats->ColumnStatistics, stats->StorageType, newSpecific); + stats->SortColumns = std::move(sortColumns); newSpecific->FullRawRowAvgSize = EstimateRowSize(*rowType, newSpecific->Format, newSpecific->Compression, false); newSpecific->FullDecodedRowAvgSize = EstimateRowSize(*rowType, newSpecific->Format, newSpecific->Compression, true); specific = newSpecific.get(); @@ -636,7 +745,10 @@ void InferStatisticsForDqSourceWrap(const TExprNode::TPtr& input, TTypeAnnotatio if (stats->Ncols == 0 || stats->Ncols > static_cast<int>(rowType->GetSize()) || stats->Nrows == 0 || stats->ByteSize == 0.0 || stats->Cost == 0.0) { auto newSpecific = std::make_shared<TS3ProviderStatistics>(*specific); + + auto sortColumns = stats->SortColumns; stats = std::make_shared<TOptimizerStatistics>(stats->Type, stats->Nrows, stats->Ncols, stats->ByteSize, stats->Cost, stats->KeyColumns, stats->ColumnStatistics, stats->StorageType, newSpecific); + stats->SortColumns = std::move(sortColumns); if (stats->Nrows == 0 && newSpecific->FullRawRowAvgSize) { stats->Nrows = newSpecific->RawByteSize / newSpecific->FullRawRowAvgSize; @@ -712,17 +824,17 @@ bool TKqpStatisticsTransformer::BeforeLambdasSpecific(const TExprNode::TPtr& inp bool matched = true; // KQP Matchers if(TKqlReadTableIndexRanges::Match(input.Get())) { - InferStatisticsForReadTableIndexRanges(input, TypeCtx); + InferStatisticsForReadTableIndexRanges(input, TypeCtx, KqpCtx); } else if(TKqlReadTableBase::Match(input.Get()) || TKqlReadTableRangesBase::Match(input.Get())){ InferStatisticsForReadTable(input, TypeCtx, KqpCtx); } - else if(TKqlLookupTableBase::Match(input.Get())) { - InferStatisticsForLookupTable(input, TypeCtx); - } else if(TKqlLookupIndexBase::Match(input.Get())){ InferStatisticsForIndexLookup(input, TypeCtx); } + else if(TKqlLookupTableBase::Match(input.Get())) { + InferStatisticsForLookupTable(input, TypeCtx); + } else if(TKqpTable::Match(input.Get())) { InferStatisticsForKqpTable(input, TypeCtx, KqpCtx); } @@ -732,6 +844,9 @@ bool TKqpStatisticsTransformer::BeforeLambdasSpecific(const TExprNode::TPtr& inp else if (TKqpCnStreamLookup::Match(input.Get())) { InferStatisticsForSteamLookup(input, TypeCtx); } + else if (TKqlIndexLookupJoinBase::Match(input.Get())) { + InferStatisticsForLookupJoin(input, TypeCtx); + } // Match a result binding atom and connect it to a stage else if(TCoParameter::Match(input.Get())) { diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy.cpp index 33151349c5..e52c874246 100644 --- a/ydb/core/kqp/opt/physical/kqp_opt_phy.cpp +++ b/ydb/core/kqp/opt/physical/kqp_opt_phy.cpp @@ -37,6 +37,7 @@ public: AddHandler(0, &TKqlReadTableRanges::Match, HNDL(BuildReadTableRangesStage)); AddHandler(0, &TKqlLookupTable::Match, HNDL(BuildLookupTableStage)); AddHandler(0, &TKqlStreamLookupTable::Match, HNDL(BuildStreamLookupTableStages)); + AddHandler(0, &TKqlIndexLookupJoin::Match, HNDL(BuildStreamIdxLookupJoinStagesKeepSorted)); AddHandler(0, &TKqlIndexLookupJoin::Match, HNDL(BuildStreamIdxLookupJoinStages)); AddHandler(0, &TKqlSequencer::Match, HNDL(BuildSequencerStages)); AddHandler(0, [](auto) { return true; }, HNDL(RemoveRedundantSortByPk)); @@ -55,6 +56,7 @@ public: AddHandler(0, &TCoFinalizeByKey::Match, HNDL(BuildFinalizeByKeyStage<false>)); AddHandler(0, &TCoShuffleByKeys::Match, HNDL(BuildShuffleStage<false>)); AddHandler(0, &TCoPartitionByKey::Match, HNDL(BuildPartitionStage<false>)); + AddHandler(0, &TCoTopBase::Match, HNDL(BuildTopStageRemoveSort<false>)); AddHandler(0, &TCoTop::Match, HNDL(BuildTopStage<false>)); AddHandler(0, &TCoTopSort::Match, HNDL(BuildTopSortStage<false>)); AddHandler(0, &TCoTakeBase::Match, HNDL(BuildTakeSkipStage<false>)); @@ -103,6 +105,7 @@ public: AddHandler(1, &TCoFinalizeByKey::Match, HNDL(BuildFinalizeByKeyStage<true>)); AddHandler(1, &TCoShuffleByKeys::Match, HNDL(BuildShuffleStage<true>)); AddHandler(1, &TCoPartitionByKey::Match, HNDL(BuildPartitionStage<true>)); + AddHandler(1, &TCoTopBase::Match, HNDL(BuildTopStageRemoveSort<true>)); AddHandler(1, &TCoTop::Match, HNDL(BuildTopStage<true>)); AddHandler(1, &TCoTopSort::Match, HNDL(BuildTopSortStage<true>)); AddHandler(1, &TCoTakeBase::Match, HNDL(BuildTakeSkipStage<true>)); @@ -183,6 +186,13 @@ protected: return output; } + TMaybeNode<TExprBase> BuildStreamIdxLookupJoinStagesKeepSorted(TExprBase node, TExprContext& ctx) { + bool ruleEnabled = KqpCtx.Config->OrderPreservingLookupJoinEnabled(); + TExprBase output = KqpBuildStreamIdxLookupJoinStagesKeepSorted(node, ctx, TypesCtx, ruleEnabled); + DumpAppliedRule("BuildStreamIdxLookupJoinStagesKeepSorted", node.Ptr(), output.Ptr(), ctx); + return output; + } + TMaybeNode<TExprBase> BuildStreamIdxLookupJoinStages(TExprBase node, TExprContext& ctx) { TExprBase output = KqpBuildStreamIdxLookupJoinStages(node, ctx); DumpAppliedRule("BuildStreamIdxLookupJoinStages", node.Ptr(), output.Ptr(), ctx); @@ -348,6 +358,15 @@ protected: DumpAppliedRule("BuildPartitionStage", node.Ptr(), output.Ptr(), ctx); return output; } + template <bool IsGlobal> + TMaybeNode<TExprBase> BuildTopStageRemoveSort(TExprBase node, TExprContext& ctx, + IOptimizationContext& optCtx, const TGetParents& getParents) + { + bool ruleEnabled = KqpCtx.Config->OrderPreservingLookupJoinEnabled(); + TExprBase output = KqpBuildTopStageRemoveSort(node, ctx, optCtx, TypesCtx, *getParents(), IsGlobal, ruleEnabled); + DumpAppliedRule("BuildTopStageRemoveSort", node.Ptr(), output.Ptr(), ctx); + return output; + } template <bool IsGlobal> TMaybeNode<TExprBase> BuildTopStage(TExprBase node, TExprContext& ctx, diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp index 8b09b77ca7..9d7668e3d3 100644 --- a/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp +++ b/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp @@ -12,6 +12,9 @@ #include <ydb/library/yql/dq/type_ann/dq_type_ann.h> #include <yql/essentials/core/yql_opt_utils.h> +#include <yql/essentials/utils/log/log.h> + + namespace NKikimr::NKqp::NOpt { using namespace NYql; @@ -764,7 +767,150 @@ NYql::NNodes::TExprBase KqpBuildStreamLookupTableStages(NYql::NNodes::TExprBase .Build().Done(); } +NYql::NNodes::TExprBase KqpBuildStreamIdxLookupJoinStagesKeepSorted(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx, + TTypeAnnotationContext& typeCtx, bool ruleEnabled) +{ + if (!ruleEnabled) { + return node; + } + + if (!node.Maybe<TKqlIndexLookupJoin>()) { + return node; + } + + const auto& idxLookupJoin = node.Cast<TKqlIndexLookupJoin>(); + + if (!idxLookupJoin.Input().Maybe<TDqCnUnionAll>()) { + return node; + } + + auto unionAll = idxLookupJoin.Input().Cast<TDqCnUnionAll>(); + auto inputStats = typeCtx.GetStats(unionAll.Output().Raw()); + if (!inputStats || !inputStats->SortColumns) { + return node; + } + + auto stage = unionAll + .Output().Maybe<TDqOutput>() + .Stage().Maybe<TDqStageBase>(); + + auto streamLookup = unionAll + .Output().Maybe<TDqOutput>() + .Stage().Maybe<TDqStageBase>() + .Inputs().Item(0).Maybe<TKqpCnStreamLookup>(); + + if (!streamLookup.IsValid()) { + return node; + } + + TExprNodeList fields; + + auto tupleType = streamLookup.Cast().InputType().Cast<TCoListType>().ItemType().Cast<TCoTupleType>(); + + auto arg = Build<TCoArgument>(ctx, node.Pos()).Name("row").Done(); + TExprNodeList args; + args.push_back(arg.Ptr()); + + auto rightStruct = tupleType.Arg(1).Cast<TCoStructType>(); + + for (auto structContent : rightStruct ) { + auto attrName = structContent.Ptr()->Child(0); + auto field = Build<TCoNameValueTuple>(ctx, node.Pos()) + .Name(attrName) + .Value<TCoMember>() + .Struct<TCoNth>() + .Tuple(arg) + .Index().Value("0").Build() + .Build() + .Name(attrName) + .Build() + .Done().Ptr(); + + fields.push_back(field); + } + + auto payload = Build<TCoNameValueTuple>(ctx, node.Pos()) + .Name().Build("_payload") + .Value(arg) + .Done().Ptr(); + + fields.push_back(payload); + + auto stageLambda = stage.Cast().Program(); + + auto orderedMap = Build<TCoOrderedMap>(ctx, node.Pos()) + .Input(stageLambda.Body()) + .Lambda() + .Args(args) + .Body<TCoAsStruct>() + .Add(fields).Build() + .Build() + .Done(); + + auto builder = Build<TDqSortColumnList>(ctx, node.Pos()); + for (size_t i = 0; i < inputStats->SortColumns->Columns.size(); i++) { + auto columnName = inputStats->SortColumns->Columns[i]; + if (inputStats->SortColumns->Aliases[i] != "") { + columnName = inputStats->SortColumns->Aliases[i] + "." + columnName; + } + builder.Add<TDqSortColumn>() + .Column<TCoAtom>().Build(columnName) + .SortDirection().Build(TTopSortSettings::AscendingSort) + .Build(); + } + + auto newStage = Build<TDqStage>(ctx, node.Pos()) + .Inputs(stage.Cast().Inputs()) + .Program() + .Args(stageLambda.Args()) + .Body(orderedMap) + .Build() + .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) + .Done().Ptr(); + + auto merge = Build<TDqCnMerge>(ctx, node.Pos()) + .Output() + .Stage(newStage) + .Index().Build(0) + .Build() + .SortColumns(builder.Build().Value()) + .Done().Ptr(); + + return Build<TDqCnUnionAll>(ctx, node.Pos()) + .Output() + .Stage<TDqStage>() + .Inputs() + .Add(merge) + .Build() + .Program() + .Args({"stream_lookup_join_output"}) + .Body<TKqpIndexLookupJoin>() + .Input<TCoOrderedMap>() + .Input<TCoToStream>() + .Input("stream_lookup_join_output") + .Build() + .Lambda() + .Args({"arg"}) + .Body<TCoMember>() + .Struct("arg") + .Name().Build("_payload") + .Build() + .Build() + .Build() + .JoinType(idxLookupJoin.JoinType()) + .LeftLabel(idxLookupJoin.LeftLabel()) + .RightLabel(idxLookupJoin.RightLabel()) + .Build() + .Build() + .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) + .Build() + .Index().Build("0") + .Build() + .Done(); +} + NYql::NNodes::TExprBase KqpBuildStreamIdxLookupJoinStages(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx) { + if (!node.Maybe<TKqlIndexLookupJoin>()) { return node; } @@ -775,11 +921,13 @@ NYql::NNodes::TExprBase KqpBuildStreamIdxLookupJoinStages(NYql::NNodes::TExprBas return node; } + auto unionAll = idxLookupJoin.Input().Cast<TDqCnUnionAll>(); + return Build<TDqCnUnionAll>(ctx, node.Pos()) .Output() .Stage<TDqStage>() .Inputs() - .Add(idxLookupJoin.Input()) + .Add(unionAll) .Build() .Program() .Args({"stream_lookup_join_output"}) diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_rules.h b/ydb/core/kqp/opt/physical/kqp_opt_phy_rules.h index 688ea4c3c4..7128016625 100644 --- a/ydb/core/kqp/opt/physical/kqp_opt_phy_rules.h +++ b/ydb/core/kqp/opt/physical/kqp_opt_phy_rules.h @@ -29,11 +29,18 @@ NYql::NNodes::TExprBase KqpBuildSequencerStages(NYql::NNodes::TExprBase node, NY NYql::NNodes::TExprBase KqpBuildStreamLookupTableStages(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx); +NYql::NNodes::TExprBase KqpBuildStreamIdxLookupJoinStagesKeepSorted(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx, + NYql::TTypeAnnotationContext& typeCtx, bool ruleEnabled); + NYql::NNodes::TExprBase KqpBuildStreamIdxLookupJoinStages(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx); NYql::NNodes::TExprBase KqpRemoveRedundantSortByPk(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx, const TKqpOptimizeContext& kqpCtx); +NYql::NNodes::TExprBase KqpBuildTopStageRemoveSort(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx, + NYql::IOptimizationContext& optCtx, NYql::TTypeAnnotationContext& typeCtx, const NYql::TParentsMap& parentsMap, + bool allowStageMultiUsage, bool ruleEnabled); + NYql::NNodes::TExprBase KqpApplyLimitToReadTable(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx, const TKqpOptimizeContext& kqpCtx); diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_sort.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy_sort.cpp index 332e82de79..8326171876 100644 --- a/ydb/core/kqp/opt/physical/kqp_opt_phy_sort.cpp +++ b/ydb/core/kqp/opt/physical/kqp_opt_phy_sort.cpp @@ -5,11 +5,13 @@ #include <ydb/core/kqp/opt/kqp_opt_impl.h> #include <yql/essentials/core/yql_opt_utils.h> +#include <ydb/library/yql/dq/type_ann/dq_type_ann.h> namespace NKikimr::NKqp::NOpt { using namespace NYql; using namespace NYql::NNodes; +using namespace NYql::NDq; // Temporary solution, should be replaced with constraints // copy-past from old engine algo: https://a.yandex-team.ru/arc_vcs/yql/providers/kikimr/yql_kikimr_opt.cpp?rev=e592a5a9509952f1c29f1ec02343dd4c05fe426d#L122 @@ -102,5 +104,153 @@ TExprBase KqpRemoveRedundantSortByPk(TExprBase node, TExprContext& ctx, const TK } } +using namespace NYql::NDq; + +bool CompatibleSort(TOptimizerStatistics::TSortColumns& existingOrder, const TCoLambda& keySelector, const TExprBase& sortDirections, TVector<TString>& sortKeys) { + if (auto body = keySelector.Body().Maybe<TCoMember>()) { + auto attrRef = body.Cast().Name().StringValue(); + auto attrName = existingOrder.Columns[0]; + auto attrNameWithAlias = existingOrder.Aliases[0] + "." + attrName; + if (attrName == attrRef || attrNameWithAlias == attrRef){ + auto sortValue = sortDirections.Cast<TCoDataCtor>().Literal().Value(); + if (FromString<bool>(sortValue)) { + sortKeys.push_back(attrRef); + return true; + } + } + } + else if (auto body = keySelector.Body().Maybe<TExprList>()) { + if (body.Cast().Size() > existingOrder.Columns.size()) { + return false; + } + + bool allMatched = false; + auto dirs = sortDirections.Cast<TExprList>(); + for (size_t i=0; i < body.Cast().Size(); i++) { + allMatched = false; + auto item = body.Cast().Item(i); + if (auto member = item.Maybe<TCoMember>()) { + auto attrRef = member.Cast().Name().StringValue(); + auto attrName = existingOrder.Columns[i]; + auto attrNameWithAlias = existingOrder.Aliases[i] + "." + attrName; + if (attrName == attrRef || attrNameWithAlias == attrRef){ + auto sortValue = dirs.Item(i).Cast<TCoDataCtor>().Literal().Value(); + if (FromString<bool>(sortValue)) { + sortKeys.push_back(attrRef); + allMatched = true; + } + } + } + if (!allMatched) { + return false; + } + } + return true; + } + return false; +} + +TExprBase KqpBuildTopStageRemoveSort( + TExprBase node, + TExprContext& ctx, + IOptimizationContext& optCtx, + TTypeAnnotationContext& typeCtx, + const TParentsMap& parentsMap, + bool allowStageMultiUsage, + bool ruleEnabled +) { + if (!ruleEnabled) { + return node; + } + + if (!node.Maybe<TCoTopBase>().Input().Maybe<TDqCnUnionAll>()) { + return node; + } + + const auto top = node.Cast<TCoTopBase>(); + const auto dqUnion = top.Input().Cast<TDqCnUnionAll>(); + + // skip this rule to activate KqpRemoveRedundantSortByPk later to reduce readings count + auto stageBody = dqUnion.Output().Stage().Program().Body(); + if (stageBody.Maybe<TCoFlatMap>()) { + auto flatmap = dqUnion.Output().Stage().Program().Body().Cast<TCoFlatMap>(); + auto input = flatmap.Input(); + bool isReadTable = input.Maybe<TKqpReadTable>().IsValid(); + bool isReadTableRanges = input.Maybe<TKqpReadTableRanges>().IsValid() || input.Maybe<TKqpReadOlapTableRanges>().IsValid() ; + if (IsPassthroughFlatMap(flatmap, nullptr)) { + if (isReadTable || isReadTableRanges) { + return node; + } + } + } else if ( + stageBody.Maybe<TKqpReadTable>().IsValid() || + stageBody.Maybe<TKqpReadTableRanges>().IsValid() || + stageBody.Maybe<TKqpReadOlapTableRanges>().IsValid() + ) { + return node; + } + + auto inputStats = typeCtx.GetStats(dqUnion.Output().Raw()); + + if (!inputStats || !inputStats->SortColumns) { + return node; + } + + if (!IsSingleConsumerConnection(dqUnion, parentsMap, allowStageMultiUsage)) { + return node; + } + + if (!CanPushDqExpr(top.Count(), dqUnion) || !CanPushDqExpr(top.KeySelectorLambda(), dqUnion)) { + return node; + } + + if (auto connToPushableStage = DqBuildPushableStage(dqUnion, ctx)) { + return TExprBase(ctx.ChangeChild(*node.Raw(), TCoTop::idx_Input, std::move(connToPushableStage))); + } + + const auto sortKeySelector = top.KeySelectorLambda(); + const auto sortDirections = top.SortDirections(); + TVector<TString> sortKeys; + + if (!CompatibleSort(*inputStats->SortColumns, sortKeySelector, sortDirections, sortKeys)) { + return node; + } + + auto builder = Build<TDqSortColumnList>(ctx, node.Pos()); + for (auto columnName : sortKeys ) { + builder.Add<TDqSortColumn>() + .Column<TCoAtom>().Build(columnName) + .SortDirection().Build(TTopSortSettings::AscendingSort) + .Build(); + } + auto columnList = builder.Build().Value(); + + return Build<TDqCnUnionAll>(ctx, node.Pos()) + .Output() + .Stage<TDqStage>() + .Inputs() + .Add<TDqCnMerge>() + .Output() + .Stage(dqUnion.Output().Stage()) + .Index(dqUnion.Output().Index()) + .Build() + .SortColumns(columnList) + .Build() + .Build() + .Program() + .Args({"stream"}) + .Body<TCoTake>() + .Input("stream") + .Count(top.Count()) + .Build() + .Build() + .Settings(NDq::TDqStageSettings::New().BuildNode(ctx, top.Pos())) + .Build() + .Index().Build(0U) + .Build() + //.SortColumns(columnList) + .Done(); +} + } // namespace NKikimr::NKqp::NOpt diff --git a/ydb/core/kqp/ut/cost/kqp_cost_ut.cpp b/ydb/core/kqp/ut/cost/kqp_cost_ut.cpp index e12223f47a..81318f827d 100644 --- a/ydb/core/kqp/ut/cost/kqp_cost_ut.cpp +++ b/ydb/core/kqp/ut/cost/kqp_cost_ut.cpp @@ -24,7 +24,7 @@ static NKikimrConfig::TAppConfig GetAppConfig(bool scanSourceRead = false, bool static NYdb::NTable::TExecDataQuerySettings GetDataQuerySettings() { NYdb::NTable::TExecDataQuerySettings execSettings; - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + execSettings.CollectQueryStats(ECollectQueryStatsMode::Full); return execSettings; } @@ -102,10 +102,10 @@ Y_UNIT_TEST_SUITE(KqpCost) { runtime->SetLogPriority(NKikimrServices::KQP_GATEWAY, NActors::NLog::PRI_DEBUG); runtime->SetLogPriority(NKikimrServices::KQP_RESOURCE_MANAGER, NActors::NLog::PRI_DEBUG); //runtime->SetLogPriority(NKikimrServices::LONG_TX_SERVICE, NActors::NLog::PRI_DEBUG); - runtime->SetLogPriority(NKikimrServices::TX_COLUMNSHARD, NActors::NLog::PRI_TRACE); - runtime->SetLogPriority(NKikimrServices::TX_COLUMNSHARD_SCAN, NActors::NLog::PRI_DEBUG); - runtime->SetLogPriority(NKikimrServices::TX_CONVEYOR, NActors::NLog::PRI_DEBUG); - runtime->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_DEBUG); + // runtime->SetLogPriority(NKikimrServices::TX_COLUMNSHARD, NActors::NLog::PRI_TRACE); + // runtime->SetLogPriority(NKikimrServices::TX_COLUMNSHARD_SCAN, NActors::NLog::PRI_DEBUG); + // runtime->SetLogPriority(NKikimrServices::TX_CONVEYOR, NActors::NLog::PRI_DEBUG); + // runtime->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_DEBUG); //runtime->SetLogPriority(NKikimrServices::BLOB_CACHE, NActors::NLog::PRI_DEBUG); //runtime->SetLogPriority(NKikimrServices::GRPC_SERVER, NActors::NLog::PRI_DEBUG); } @@ -407,7 +407,7 @@ Y_UNIT_TEST_SUITE(KqpCost) { UNIT_ASSERT_VALUES_EQUAL(readsByTable.at("/Root/Join1_1").second, 136); } - Y_UNIT_TEST(RangeFullScan) { + Y_UNIT_TEST(AAARangeFullScan) { TKikimrRunner kikimr(GetAppConfig()); auto db = kikimr.GetTableClient(); @@ -422,6 +422,9 @@ Y_UNIT_TEST_SUITE(KqpCost) { auto result = session.ExecuteDataQuery(query, txControl, GetDataQuerySettings()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + Cerr << "PONOS" << Endl; + Cerr << result.GetQueryPlan() << Endl; + CompareYson(R"( [ [[3500u];["None"];[1u];["Anna"]] diff --git a/ydb/core/kqp/ut/join/data/join_order/tpcds64_1000s.json b/ydb/core/kqp/ut/join/data/join_order/tpcds64_1000s.json index 9f04a9dfad..8b84f0fc0b 100644 --- a/ydb/core/kqp/ut/join/data/join_order/tpcds64_1000s.json +++ b/ydb/core/kqp/ut/join/data/join_order/tpcds64_1000s.json @@ -1,5 +1,5 @@ { - "op_name":"InnerJoin (MapJoin)", + "op_name":"InnerJoin (Grace)", "args": [ { diff --git a/ydb/core/kqp/ut/join/data/queries/general_priorities_bug.sql b/ydb/core/kqp/ut/join/data/queries/general_priorities_bug.sql new file mode 100644 index 0000000000..bbc555b6c2 --- /dev/null +++ b/ydb/core/kqp/ut/join/data/queries/general_priorities_bug.sql @@ -0,0 +1,16 @@ +PRAGMA ydb.EnableOrderPreservingLookupJoin="true"; +PRAGMA ydb.CostBasedOptimizationLevel="1"; + +SELECT + doc.id AS document_id +FROM ( +SELECT d.id AS id, d.exec_dt AS exec_dt +FROM `/Root/bank_document` VIEW ix_bank_document_exec_dt_accounts AS d +LEFT JOIN `/Root/bank_sub_document` VIEW IX_BANK_SUB_DOCUMENT_DOCUMENT_ID AS sd + ON d.id = sd.document_id +WHERE sd.document_id IS NULL + AND d.exec_dt >= Cast('1990-12-10' as Date) + AND d.acc_dt_id = 15 +ORDER BY exec_dt, id +LIMIT 1000 +) AS doc;
\ No newline at end of file diff --git a/ydb/core/kqp/ut/join/data/queries/general_priorities_bug2.sql b/ydb/core/kqp/ut/join/data/queries/general_priorities_bug2.sql new file mode 100644 index 0000000000..d884f56499 --- /dev/null +++ b/ydb/core/kqp/ut/join/data/queries/general_priorities_bug2.sql @@ -0,0 +1,20 @@ +PRAGMA ydb.EnableOrderPreservingLookupJoin="true"; +PRAGMA ydb.CostBasedOptimizationLevel="1"; + + +SELECT + doc.id AS document_id +FROM ( +SELECT d.id AS id, d.exec_dt AS exec_dt +FROM `/Root/bank_document` VIEW ix_bank_document_exec_dt_accounts AS d +LEFT JOIN `/Root/bank_sub_document` VIEW IX_BANK_SUB_DOCUMENT_DOCUMENT_ID AS sd + ON d.id = sd.document_id +LEFT JOIN `/Root/bank_sub_document` VIEW IX_BANK_SUB_DOCUMENT_DOCUMENT_ID AS sdd + ON d.id = sdd.document_id +WHERE sd.document_id IS NULL + AND d.exec_dt >= Cast('1990-12-10' as Date) + AND d.acc_dt_id = 15 +--ORDER BY exec_dt, id +ORDER BY d.exec_dt, id +LIMIT 1000 +) AS doc;
\ No newline at end of file diff --git a/ydb/core/kqp/ut/join/data/queries/general_priorities_bug3.sql b/ydb/core/kqp/ut/join/data/queries/general_priorities_bug3.sql new file mode 100644 index 0000000000..99f5165f5b --- /dev/null +++ b/ydb/core/kqp/ut/join/data/queries/general_priorities_bug3.sql @@ -0,0 +1,16 @@ +PRAGMA ydb.EnableOrderPreservingLookupJoin="true"; +PRAGMA ydb.CostBasedOptimizationLevel="1"; + +SELECT + doc.id AS document_id +FROM ( +SELECT d.id AS id, d.exec_dt AS exec_dt +FROM `/Root/bank_document` VIEW ix_bank_document_exec_dt_accounts AS d +LEFT JOIN `/Root/bank_sub_document` AS sd + ON d.id = sd.document_id +WHERE sd.document_id IS NULL + AND d.exec_dt >= Cast('1990-12-10' as Date) + AND d.acc_dt_id = 15 +ORDER BY exec_dt, id +LIMIT 1000 +) AS doc;
\ No newline at end of file diff --git a/ydb/core/kqp/ut/join/data/queries/general_priorities_bug4.sql b/ydb/core/kqp/ut/join/data/queries/general_priorities_bug4.sql new file mode 100644 index 0000000000..6ed51717a1 --- /dev/null +++ b/ydb/core/kqp/ut/join/data/queries/general_priorities_bug4.sql @@ -0,0 +1,20 @@ +PRAGMA ydb.EnableOrderPreservingLookupJoin="true"; +PRAGMA ydb.CostBasedOptimizationLevel="1"; + + +SELECT + doc.id AS document_id +FROM ( +SELECT d.id AS id, d.exec_dt AS exec_dt +FROM `/Root/bank_document` VIEW ix_bank_document_exec_dt_accounts AS d +LEFT JOIN `/Root/bank_sub_document` AS sd + ON d.id = sd.document_id +LEFT JOIN `/Root/bank_sub_document` AS sdd + ON d.id = sdd.document_id +WHERE sd.document_id IS NULL + AND d.exec_dt >= Cast('1990-12-10' as Date) + AND d.acc_dt_id = 15 +--ORDER BY exec_dt, id +ORDER BY d.exec_dt, id +LIMIT 1000 +) AS doc;
\ No newline at end of file diff --git a/ydb/core/kqp/ut/join/data/schema/general_priorities_bug.sql b/ydb/core/kqp/ut/join/data/schema/general_priorities_bug.sql new file mode 100644 index 0000000000..16a1e8034d --- /dev/null +++ b/ydb/core/kqp/ut/join/data/schema/general_priorities_bug.sql @@ -0,0 +1,16 @@ +CREATE TABLE `/Root/bank_document` ( + id Int32 NOT NULL, + exec_dt Date NOT NULL, + acc_dt_id Int32 NOT NULL, + PRIMARY KEY (id), + INDEX ix_bank_document_exec_dt_accounts GLOBAL ON (exec_dt, id) COVER ( acc_dt_id ) +); + +CREATE TABLE `/Root/bank_sub_document` ( + document_id Int32 NOT NULL, + blah String NOT NULL, + blah_blah String NOT NULL, + blah2 String NOT NULL, + PRIMARY KEY (document_id, blah), + INDEX IX_BANK_SUB_DOCUMENT_DOCUMENT_ID GLOBAL ON (document_id) COVER ( blah2 ) +);
\ No newline at end of file diff --git a/ydb/core/kqp/ut/join/data/stats/general_priorities_bug.json b/ydb/core/kqp/ut/join/data/stats/general_priorities_bug.json new file mode 100644 index 0000000000..5744ddb372 --- /dev/null +++ b/ydb/core/kqp/ut/join/data/stats/general_priorities_bug.json @@ -0,0 +1,17 @@ +{ + "/Root/bank_document": { + "n_rows": 10, + "byte_size": 100 }, + "/Root/bank_document/ix_bank_document_exec_dt_accounts/indexImplTable": { + "n_rows": 10, + "byte_size": 100 }, + "/Root/bank_sub_document": { + "n_rows": 100, + "byte_size": 100, + "key_columns" : [] + }, + "/Root/bank_sub_document/IX_BANK_SUB_DOCUMENT_DOCUMENT_ID/indexImplTable": { + "n_rows": 100, + "byte_size": 100 + } +}
\ No newline at end of file diff --git a/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp index 4c7f59dd50..c36e8bc527 100644 --- a/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_order_ut.cpp @@ -87,6 +87,8 @@ static void CreateSampleTable(NYdb::NQuery::TSession session, bool useColumnStor CreateTables(session, "schema/lookupbug.sql", useColumnStore); + CreateTables(session, "schema/general_priorities_bug.sql", useColumnStore); + CreateView(session, "view/tpch_random_join_view.sql"); } @@ -95,6 +97,10 @@ static TKikimrRunner GetKikimrWithJoinSettings(bool useStreamLookupJoin = false, NKikimrKqp::TKqpSetting setting; + setting.SetName("EnableKqpDataQueryStreamLookup"); + setting.SetValue("true"); + settings.push_back(setting); + if (stats != "") { setting.SetName("OptOverrideStatistics"); setting.SetValue(stats); @@ -362,7 +368,7 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { TChainTester(65).Test(); } - TString ExecuteJoinOrderTestDataQueryWithStats(const TString& queryPath, const TString& statsPath, bool useStreamLookupJoin, bool useColumnStore, bool useCBO = true) { + TString ExecuteJoinOrderTestGenericQueryWithStats(const TString& queryPath, const TString& statsPath, bool useStreamLookupJoin, bool useColumnStore, bool useCBO = true) { auto kikimr = GetKikimrWithJoinSettings(useStreamLookupJoin, GetStatic(statsPath), useCBO); kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableViews(true); auto db = kikimr.GetQueryClient(); @@ -420,95 +426,95 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(FiveWayJoin, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats( + ExecuteJoinOrderTestGenericQueryWithStats( "queries/five_way_join.sql", "stats/basic.json", StreamLookupJoin, ColumnStore ); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(FiveWayJoinStatsOverride, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats( + ExecuteJoinOrderTestGenericQueryWithStats( "queries/five_way_join_stats_override.sql", "stats/basic.json", StreamLookupJoin, ColumnStore ); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(FourWayJoinLeftFirst, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats( + ExecuteJoinOrderTestGenericQueryWithStats( "queries/four_way_join_left_first.sql", "stats/basic.json", StreamLookupJoin, ColumnStore ); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(FiveWayJoinWithPreds, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats( + ExecuteJoinOrderTestGenericQueryWithStats( "queries/five_way_join_with_preds.sql", "stats/basic.json", StreamLookupJoin, ColumnStore ); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(FiveWayJoinWithComplexPreds, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats( + ExecuteJoinOrderTestGenericQueryWithStats( "queries/five_way_join_with_complex_preds.sql", "stats/basic.json", StreamLookupJoin, ColumnStore ); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(FiveWayJoinWithComplexPreds2, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats( + ExecuteJoinOrderTestGenericQueryWithStats( "queries/five_way_join_with_complex_preds2.sql", "stats/basic.json", StreamLookupJoin, ColumnStore ); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(FiveWayJoinWithPredsAndEquiv, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats( + ExecuteJoinOrderTestGenericQueryWithStats( "queries/four_way_join_with_preds_and_equiv.sql", "stats/basic.json", StreamLookupJoin, ColumnStore ); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(FourWayJoinWithPredsAndEquivAndLeft, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats( + ExecuteJoinOrderTestGenericQueryWithStats( "queries/four_way_join_with_preds_and_equiv_and_left.sql", "stats/basic.json", StreamLookupJoin, ColumnStore ); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(FiveWayJoinWithConstantFold, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/five_way_join_with_constant_fold.sql", "stats/basic.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/five_way_join_with_constant_fold.sql", "stats/basic.json", StreamLookupJoin, ColumnStore); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(FiveWayJoinWithConstantFoldOpt, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/five_way_join_with_constant_fold_opt.sql", "stats/basic.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/five_way_join_with_constant_fold_opt.sql", "stats/basic.json", StreamLookupJoin, ColumnStore); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(DatetimeConstantFold, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/datetime_constant_fold.sql", "stats/basic.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/datetime_constant_fold.sql", "stats/basic.json", StreamLookupJoin, ColumnStore); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCHRandomJoinViewJustWorks, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/tpch_random_join_view_just_works.sql", "stats/tpch1000s.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/tpch_random_join_view_just_works.sql", "stats/tpch1000s.json", StreamLookupJoin, ColumnStore); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCH3, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/tpch3.sql", "stats/tpch1000s.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/tpch3.sql", "stats/tpch1000s.json", StreamLookupJoin, ColumnStore); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCH5, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/tpch5.sql", "stats/tpch1000s.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/tpch5.sql", "stats/tpch1000s.json", StreamLookupJoin, ColumnStore); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCH8, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/tpch8.sql", "stats/tpch100s.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/tpch8.sql", "stats/tpch100s.json", StreamLookupJoin, ColumnStore); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCH10, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/tpch10.sql", "stats/tpch1000s.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/tpch10.sql", "stats/tpch1000s.json", StreamLookupJoin, ColumnStore); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCH11, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/tpch11.sql", "stats/tpch1000s.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/tpch11.sql", "stats/tpch1000s.json", StreamLookupJoin, ColumnStore); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCH21, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/tpch21.sql", "stats/tpch1000s.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/tpch21.sql", "stats/tpch1000s.json", StreamLookupJoin, ColumnStore); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCDS16, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/tpcds16.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/tpcds16.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); } /* tpcds23 has > 1 result sets */ @@ -518,40 +524,64 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { ); } + bool CheckLimitOnlyNotTopSort(const TString& plan) { + return plan.Contains("Limit") && !plan.Contains("Top"); + } + + Y_UNIT_TEST(GeneralPrioritiesBug1) { + auto plan = ExecuteJoinOrderTestGenericQueryWithStats("queries/general_priorities_bug.sql", "stats/general_priorities_bug.json", true, false); + UNIT_ASSERT(CheckLimitOnlyNotTopSort(plan)); + } + + Y_UNIT_TEST(GeneralPrioritiesBug2) { + auto plan = ExecuteJoinOrderTestGenericQueryWithStats("queries/general_priorities_bug2.sql", "stats/general_priorities_bug.json", true, false); + UNIT_ASSERT(CheckLimitOnlyNotTopSort(plan)); + } + + Y_UNIT_TEST(GeneralPrioritiesBug3) { + auto plan = ExecuteJoinOrderTestGenericQueryWithStats("queries/general_priorities_bug3.sql", "stats/general_priorities_bug.json", true, false); + UNIT_ASSERT(CheckLimitOnlyNotTopSort(plan)); + } + + Y_UNIT_TEST(GeneralPrioritiesBug4) { + auto plan = ExecuteJoinOrderTestGenericQueryWithStats("queries/general_priorities_bug4.sql", "stats/general_priorities_bug.json", true, false); + UNIT_ASSERT(CheckLimitOnlyNotTopSort(plan)); + } + Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCDS34, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/tpcds34.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/tpcds34.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCDS61, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/tpcds61.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/tpcds61.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCDS87, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/tpcds87.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/tpcds87.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCDS88, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/tpcds88.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/tpcds88.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCDS90, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/tpcds90.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/tpcds90.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCDS92, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/tpcds92.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/tpcds92.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCDS94, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/tpcds94.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/tpcds94.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCDS95, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/tpcds95.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/tpcds95.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TPCDS96, StreamLookupJoin, ColumnStore) { - ExecuteJoinOrderTestDataQueryWithStats("queries/tpcds96.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); + ExecuteJoinOrderTestGenericQueryWithStats("queries/tpcds96.sql", "stats/tpcds1000s.json", StreamLookupJoin, ColumnStore); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TestJoinHint1, StreamLookupJoin, ColumnStore) { @@ -615,7 +645,7 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { }; Y_UNIT_TEST(OltpJoinTypeHintCBOTurnOFF) { - auto plan = ExecuteJoinOrderTestDataQueryWithStats("queries/oltp_join_type_hint_cbo_turnoff.sql", "stats/basic.json", false, false, false); + auto plan = ExecuteJoinOrderTestGenericQueryWithStats("queries/oltp_join_type_hint_cbo_turnoff.sql", "stats/basic.json", false, false, false); auto detailedPlan = GetDetailedJoinOrder(plan); auto joinFinder = TFindJoinWithLabels(detailedPlan); @@ -626,23 +656,25 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) { } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TestJoinOrderHintsSimple, StreamLookupJoin, ColumnStore) { - auto plan = ExecuteJoinOrderTestDataQueryWithStats("queries/join_order_hints_simple.sql", "stats/basic.json", StreamLookupJoin, ColumnStore); + auto plan = ExecuteJoinOrderTestGenericQueryWithStats("queries/join_order_hints_simple.sql", "stats/basic.json", StreamLookupJoin, ColumnStore); UNIT_ASSERT_VALUES_EQUAL(GetJoinOrder(plan).GetStringRobust(), R"(["T",["R","S"]])") ; } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TestJoinOrderHintsComplex, StreamLookupJoin, ColumnStore) { - auto plan = ExecuteJoinOrderTestDataQueryWithStats("queries/join_order_hints_complex.sql", "stats/basic.json", StreamLookupJoin, ColumnStore); + auto plan = ExecuteJoinOrderTestGenericQueryWithStats("queries/join_order_hints_complex.sql", "stats/basic.json", StreamLookupJoin, ColumnStore); auto joinOrder = GetJoinOrder(plan).GetStringRobust(); UNIT_ASSERT_C(joinOrder.find(R"([["R","S"],["T","U"]])") != TString::npos, joinOrder); } Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TestJoinOrderHintsManyHintTrees, StreamLookupJoin, ColumnStore) { - auto plan = ExecuteJoinOrderTestDataQueryWithStats("queries/join_order_hints_many_hint_trees.sql", "stats/basic.json", StreamLookupJoin, ColumnStore); + auto plan = ExecuteJoinOrderTestGenericQueryWithStats("queries/join_order_hints_many_hint_trees.sql", "stats/basic.json", StreamLookupJoin, ColumnStore); auto joinOrder = GetJoinOrder(plan).GetStringRobust(); UNIT_ASSERT_C(joinOrder.find(R"(["R","S"])") != TString::npos, joinOrder); UNIT_ASSERT_C(joinOrder.find(R"(["T","U"])") != TString::npos, joinOrder); } + + void CanonizedJoinOrderTest(const TString& queryPath, const TString& statsPath, TString correctJoinOrderPath, bool useStreamLookupJoin, bool useColumnStore ) { auto kikimr = GetKikimrWithJoinSettings(useStreamLookupJoin, GetStatic(statsPath)); diff --git a/ydb/core/kqp/ut/query/kqp_explain_ut.cpp b/ydb/core/kqp/ut/query/kqp_explain_ut.cpp index 2b913c9c5c..1c8f4e5cd0 100644 --- a/ydb/core/kqp/ut/query/kqp_explain_ut.cpp +++ b/ydb/core/kqp/ut/query/kqp_explain_ut.cpp @@ -593,7 +593,8 @@ Y_UNIT_TEST_SUITE(KqpExplain) { NJson::ReadJsonTree(*res.PlanJson, &plan, true); UNIT_ASSERT(ValidatePlanNodeIds(plan)); - auto join = FindPlanNodeByKv(plan, "Node Type", "FullJoin (JoinDict)"); + Cout << plan.GetStringRobust() << Endl; + auto join = FindPlanNodeByKv(plan, "Node Type", "FullJoin (Grace)"); UNIT_ASSERT(join.IsDefined()); auto left = FindPlanNodeByKv(join, "Table", "EightShard"); UNIT_ASSERT(left.IsDefined()); diff --git a/ydb/library/yql/dq/opt/dq_opt_phy.h b/ydb/library/yql/dq/opt/dq_opt_phy.h index 4711cb908f..693beec611 100644 --- a/ydb/library/yql/dq/opt/dq_opt_phy.h +++ b/ydb/library/yql/dq/opt/dq_opt_phy.h @@ -72,6 +72,9 @@ NNodes::TExprBase DqBuildFinalizeByKeyStage(NNodes::TExprBase node, TExprContext NNodes::TExprBase DqBuildAggregationResultStage(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx); +NNodes::TExprBase DqBuildTopStageRemoveSort(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, + TTypeAnnotationContext& typeCtx, const TParentsMap& parentsMap, bool allowStageMultiUsage = true); + NNodes::TExprBase DqBuildTopStage(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TParentsMap& parentsMap, bool allowStageMultiUsage = true); diff --git a/ydb/library/yql/dq/opt/dq_opt_stat.cpp b/ydb/library/yql/dq/opt/dq_opt_stat.cpp index 4ddd5e7995..be606b9d0b 100644 --- a/ydb/library/yql/dq/opt/dq_opt_stat.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_stat.cpp @@ -246,7 +246,6 @@ bool IsConstantExprWithParams(const TExprNode::TPtr& input) { return false; } - /** * Compute statistics for map join * FIX: Currently we treat all join the same from the cost perspective, need to refine cost function @@ -483,6 +482,7 @@ void InferStatisticsForFlatMap(const TExprNode::TPtr& input, TTypeAnnotationCont inputStats->ColumnStatistics, inputStats->StorageType); + outputStats.SortColumns = inputStats->SortColumns; outputStats.Labels = inputStats->Labels; outputStats.Selectivity *= (inputStats->Selectivity * selectivity); @@ -534,7 +534,9 @@ void InferStatisticsForFilter(const TExprNode::TPtr& input, TTypeAnnotationConte inputStats->Cost, inputStats->KeyColumns, inputStats->ColumnStatistics, - inputStats->StorageType); + inputStats->StorageType + ); + outputStats.SortColumns = inputStats->SortColumns; outputStats.Selectivity *= (selectivity * inputStats->Selectivity); outputStats.Labels = inputStats->Labels; @@ -562,24 +564,6 @@ void InferStatisticsForSkipNullMembers(const TExprNode::TPtr& input, TTypeAnnota } /** - * Infer statistics and costs for ExtractlMembers - * We just return the input statistics. -*/ -void InferStatisticsForExtractMembers(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx) { - - auto inputNode = TExprBase(input); - auto extractMembers = inputNode.Cast<TCoExtractMembers>(); - auto extractMembersInput = extractMembers.Input(); - - auto inputStats = typeCtx->GetStats(extractMembersInput.Raw() ); - if (!inputStats) { - return; - } - - typeCtx->SetStats( input.Get(), inputStats ); -} - -/** * Infer statistics and costs for AggregateCombine * We just return the input statistics. */ @@ -594,7 +578,7 @@ void InferStatisticsForAggregateCombine(const TExprNode::TPtr& input, TTypeAnnot return; } - typeCtx->SetStats( input.Get(), inputStats ); + typeCtx->SetStats( input.Get(), RemoveOrdering(inputStats)); } /** @@ -719,4 +703,111 @@ void InferStatisticsForStage(const TExprNode::TPtr& input, TTypeAnnotationContex } } +void InferStatisticsForDqMerge(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx) { + auto inputNode = TExprBase(input); + auto merge = inputNode.Cast<TDqCnMerge>(); + + auto inputStats = typeCtx->GetStats(merge.Output().Raw()); + if (!inputStats) { + return; + } + + auto newStats = RemoveOrdering(inputStats); + + auto sortedPrefixPtr = TIntrusivePtr<TOptimizerStatistics::TSortColumns>(); + + TVector<TString> sortedPrefixCols; + TVector<TString> sortedPrefixAliases; + + for ( auto c : merge.SortColumns() ) { + auto column = c.Column().StringValue(); + auto sortDir = c.SortDirection().StringValue(); + + if (sortDir != "Asc") { + break; + } + + auto alias = ExtractAlias(column); + auto columnNoAlias = RemoveAliases(column); + + sortedPrefixCols.push_back(columnNoAlias); + sortedPrefixAliases.push_back(alias); + } + + if (sortedPrefixCols.size()) { + sortedPrefixPtr = TIntrusivePtr<TOptimizerStatistics::TSortColumns>(new TOptimizerStatistics::TSortColumns(sortedPrefixCols, sortedPrefixAliases)); + } + + newStats->SortColumns = sortedPrefixPtr; + YQL_CLOG(TRACE, CoreDq) << "Infer statistics for Merge: " << newStats->ToString(); + + typeCtx->SetStats(merge.Raw(), newStats); +} + +/** + * Just update the sorted order with alias + */ +void InferStatisticsForDqPhyCrossJoin(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx) { + auto inputNode = TExprBase(input); + auto cross = inputNode.Cast<TDqPhyCrossJoin>(); + + auto inputStats = typeCtx->GetStats(cross.LeftInput().Raw()); + if (!inputStats) { + return; + } + + auto sortedPrefix = inputStats->SortColumns; + TString aliasName = ""; + if (auto leftLabel = cross.LeftLabel().Maybe<TCoAtom>()) { + aliasName = leftLabel.Cast().StringValue(); + } + + TVector<TString> sortedPrefixCols; + TVector<TString> sortedPrefixAliases; + + if (sortedPrefix) { + sortedPrefixCols = sortedPrefix->Columns; + sortedPrefixAliases = sortedPrefix->Aliases; + if (aliasName != "") { + for (size_t i=0; i<sortedPrefix->Aliases.size(); i++) { + sortedPrefixAliases[i] = aliasName; + } + } + } + + auto sortedPrefixPtr = TIntrusivePtr<TOptimizerStatistics::TSortColumns>(); + if (sortedPrefixCols.size()) { + sortedPrefixPtr = TIntrusivePtr<TOptimizerStatistics::TSortColumns>(new TOptimizerStatistics::TSortColumns(sortedPrefixCols, sortedPrefixAliases)); + } + + auto outputStats = RemoveOrdering(inputStats); + outputStats->SortColumns = sortedPrefixPtr; + typeCtx->SetStats(cross.Raw(), outputStats); +} + + + +std::shared_ptr<TOptimizerStatistics> RemoveOrdering(const std::shared_ptr<TOptimizerStatistics>& stats) { + if (stats->SortColumns) { + auto newStats = *stats; + newStats.SortColumns = TIntrusivePtr<TOptimizerStatistics::TSortColumns>(); + return std::make_shared<TOptimizerStatistics>(std::move(newStats)); + } else { + return stats; + } +} + +std::shared_ptr<TOptimizerStatistics> RemoveOrdering(const std::shared_ptr<TOptimizerStatistics>& stats, const TExprNode::TPtr& input) { + if (TCoTopBase::Match(input.Get()) || + TCoSortBase::Match(input.Get()) || + TDqCnHashShuffle::Match(input.Get()) || + TDqCnBroadcast::Match(input.Get()) || + TDqCnUnionAll::Match(input.Get())) { + return RemoveOrdering(stats); + } else { + return stats; + } +} + + } // namespace NYql::NDq { diff --git a/ydb/library/yql/dq/opt/dq_opt_stat.h b/ydb/library/yql/dq/opt/dq_opt_stat.h index 0959bcbef4..d85f80e566 100644 --- a/ydb/library/yql/dq/opt/dq_opt_stat.h +++ b/ydb/library/yql/dq/opt/dq_opt_stat.h @@ -10,18 +10,21 @@ namespace NYql::NDq { void InferStatisticsForFlatMap(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx); void InferStatisticsForFilter(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx); void InferStatisticsForSkipNullMembers(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx); -void InferStatisticsForExtractMembers(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx); void InferStatisticsForAggregateCombine(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx); void InferStatisticsForAggregateMergeFinalize(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx); void PropagateStatisticsToLambdaArgument(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx); void PropagateStatisticsToStageArguments(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx); void InferStatisticsForStage(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx); void InferStatisticsForDqSource(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx); +void InferStatisticsForDqMerge(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx); void InferStatisticsForGraceJoin(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx, const IProviderContext& ctx, TCardinalityHints hints = {}); void InferStatisticsForMapJoin(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx, const IProviderContext& ctx, TCardinalityHints hints = {}); void InferStatisticsForDqJoin(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx, const IProviderContext& ctx, TCardinalityHints hints = {}); +void InferStatisticsForDqPhyCrossJoin(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx); void InferStatisticsForAsList(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx); bool InferStatisticsForListParam(const TExprNode::TPtr& input, TTypeAnnotationContext* typeCtx); +std::shared_ptr<TOptimizerStatistics> RemoveOrdering(const std::shared_ptr<TOptimizerStatistics>& stats); +std::shared_ptr<TOptimizerStatistics> RemoveOrdering(const std::shared_ptr<TOptimizerStatistics>& stats, const TExprNode::TPtr& input); class TPredicateSelectivityComputer { public: diff --git a/ydb/library/yql/dq/opt/dq_opt_stat_transformer_base.cpp b/ydb/library/yql/dq/opt/dq_opt_stat_transformer_base.cpp index 50f0b2d4a2..1b2604215d 100644 --- a/ydb/library/yql/dq/opt/dq_opt_stat_transformer_base.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_stat_transformer_base.cpp @@ -3,6 +3,9 @@ #include <ydb/library/yql/dq/opt/dq_opt_stat.h> #include <yql/essentials/core/yql_expr_optimize.h> +#include <yql/essentials/utils/log/log.h> + + namespace NYql::NDq { using namespace NNodes; @@ -43,9 +46,6 @@ bool TDqStatisticsTransformerBase::BeforeLambdas(const TExprNode::TPtr& input, T else if(TCoSkipNullMembers::Match(input.Get())){ InferStatisticsForSkipNullMembers(input, TypeCtx); } - else if(TCoExtractMembers::Match(input.Get())){ - InferStatisticsForExtractMembers(input, TypeCtx); - } else if(TCoAggregateCombine::Match(input.Get())){ InferStatisticsForAggregateCombine(input, TypeCtx); } @@ -68,6 +68,9 @@ bool TDqStatisticsTransformerBase::BeforeLambdas(const TExprNode::TPtr& input, T else if (TDqJoin::Match(input.Get())) { InferStatisticsForDqJoin(input, TypeCtx, Pctx, CardinalityHints); } + else if(TDqPhyCrossJoin::Match(input.Get())) { + InferStatisticsForDqPhyCrossJoin(input, TypeCtx); + } // Do nothing in case of EquiJoin, otherwise the EquiJoin rule won't fire else if(TCoEquiJoin::Match(input.Get())){ @@ -77,6 +80,11 @@ bool TDqStatisticsTransformerBase::BeforeLambdas(const TExprNode::TPtr& input, T else if (TDqSource::Match(input.Get())) { InferStatisticsForDqSource(input, TypeCtx); } + + // In case of DqCnMerge, update the sorted info with correct sorting + else if (TDqCnMerge::Match(input.Get())) { + InferStatisticsForDqMerge(input, TypeCtx); + } else { matched = false; } @@ -90,7 +98,7 @@ bool TDqStatisticsTransformerBase::BeforeLambdasUnmatched(const TExprNode::TPtr& if (input->ChildrenSize() >= 1) { auto stats = TypeCtx->GetStats(input->ChildRef(0).Get()); if (stats) { - TypeCtx->SetStats(input.Get(), stats); + TypeCtx->SetStats(input.Get(), RemoveOrdering(stats, input)); } } return true; @@ -112,4 +120,3 @@ bool TDqStatisticsTransformerBase::AfterLambdas(const TExprNode::TPtr& input, TE void TDqStatisticsTransformerBase::Rewind() { } } // namespace NYql::NDq - diff --git a/ydb/tests/functional/canonical/canondata/test_sql.TestCanonicalFolder1.test_case_join_group_by_lookup.script-script_/join_group_by_lookup.script.plan b/ydb/tests/functional/canonical/canondata/test_sql.TestCanonicalFolder1.test_case_join_group_by_lookup.script-script_/join_group_by_lookup.script.plan index 2e61e6e045..15e1812ce7 100644 --- a/ydb/tests/functional/canonical/canondata/test_sql.TestCanonicalFolder1.test_case_join_group_by_lookup.script-script_/join_group_by_lookup.script.plan +++ b/ydb/tests/functional/canonical/canondata/test_sql.TestCanonicalFolder1.test_case_join_group_by_lookup.script-script_/join_group_by_lookup.script.plan @@ -64,9 +64,9 @@ "Group", "Value" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "Group" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_coalesce-and-join.test_/query_1.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_coalesce-and-join.test_/query_1.plan index 4aee9fc64f..c816fcad7f 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_coalesce-and-join.test_/query_1.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_coalesce-and-join.test_/query_1.plan @@ -57,9 +57,9 @@ "Columns": [ "x" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "x" ], @@ -84,9 +84,9 @@ "Columns": [ "x" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "x" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_coalesce-and-join.test_/query_2.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_coalesce-and-join.test_/query_2.plan index cbe27e4695..6c64755ba8 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_coalesce-and-join.test_/query_2.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_coalesce-and-join.test_/query_2.plan @@ -57,9 +57,9 @@ "Columns": [ "x" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "x" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_coalesce-and-join.test_/query_3.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_coalesce-and-join.test_/query_3.plan index f803b73088..991fbb5d30 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_coalesce-and-join.test_/query_3.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_coalesce-and-join.test_/query_3.plan @@ -58,9 +58,9 @@ "x", "y" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "x" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_coalesce-and-join.test_/query_4.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_coalesce-and-join.test_/query_4.plan index 99a93e2e04..640f27323b 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_coalesce-and-join.test_/query_4.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_coalesce-and-join.test_/query_4.plan @@ -57,9 +57,9 @@ "Columns": [ "pkxx" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "pkxx" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join-group-by-with-null.test_/query_2.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join-group-by-with-null.test_/query_2.plan index 5e7076d612..84fd818795 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join-group-by-with-null.test_/query_2.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join-group-by-with-null.test_/query_2.plan @@ -82,9 +82,9 @@ "q1", "q2" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "q1" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join-group-by-with-null.test_/query_3.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join-group-by-with-null.test_/query_3.plan index f457ccde5f..0705de1031 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join-group-by-with-null.test_/query_3.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join-group-by-with-null.test_/query_3.plan @@ -82,9 +82,9 @@ "q1", "q2" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "q1" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join0.test_/query_1.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join0.test_/query_1.plan index 4595cc4da8..6a60be6db4 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join0.test_/query_1.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join0.test_/query_1.plan @@ -5,7 +5,7 @@ "Plans": [ { "Node Type": "ResultSet", - "PlanNodeId": 12, + "PlanNodeId": 15, "PlanNodeType": "ResultSet", "Plans": [ { @@ -14,18 +14,18 @@ { "Inputs": [ { - "ExternalPlanNodeId": 10 + "ExternalPlanNodeId": 13 } ], "Limit": "1001", "Name": "Limit" } ], - "PlanNodeId": 11, + "PlanNodeId": 14, "Plans": [ { "Node Type": "Merge", - "PlanNodeId": 10, + "PlanNodeId": 13, "PlanNodeType": "Connection", "Plans": [ { @@ -45,7 +45,7 @@ "Condition": "i1.q2 = i2_1.q2", "Inputs": [ { - "ExternalPlanNodeId": 8 + "ExternalPlanNodeId": 11 }, { "InternalOperatorId": 2 @@ -56,75 +56,73 @@ { "Inputs": [ { - "ExternalPlanNodeId": 5 + "ExternalPlanNodeId": 8 } ], "Name": "Filter", "Predicate": "Exist(item.i2_1.q2)" } ], - "PlanNodeId": 9, + "PlanNodeId": 12, "Plans": [ { "Node Type": "Broadcast", - "PlanNodeId": 5, + "PlanNodeId": 8, "PlanNodeType": "Connection", "Plans": [ { "Node Type": "Stage", - "PlanNodeId": 4, + "PlanNodeId": 7, "Plans": [ { "Node Type": "UnionAll", - "PlanNodeId": 3, + "PlanNodeId": 6, "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "InnerJoin (MapJoin)-Filter", - "Operators": [ - { - "Condition": "i2_1.q1 = ss.x", - "Inputs": [ - { - "InternalOperatorId": 1 - } - ], - "Name": "InnerJoin (MapJoin)" - }, - { - "Inputs": [ - { - "ExternalPlanNodeId": 1 - } - ], - "Name": "Filter", - "Predicate": "Exist(item.q1)" - } - ], - "PlanNodeId": 2, + "Node Type": "Stage", + "PlanNodeId": 5, "Plans": [ { - "Node Type": "TableFullScan", - "Operators": [ + "Node Type": "UnionAll", + "PlanNodeId": 4, + "PlanNodeType": "Connection", + "Plans": [ { - "Inputs": [], - "Name": "TableFullScan", - "ReadColumns": [ - "q1", - "q2" - ], - "ReadRanges": [ - "q1 (-\u221e, +\u221e)", - "q2 (-\u221e, +\u221e)" - ], - "ReadRangesPointPrefixLen": "0", - "Scan": "Parallel", - "Table": "postgres_jointest/join0.test_plan/int8_tbl" + "Node Type": "Collect", + "PlanNodeId": 3, + "Plans": [ + { + "Columns": [ + "q1", + "q2" + ], + "E-Cost": "0", + "E-Rows": "1", + "E-Size": "0", + "LookupKeyColumns": [ + "q1" + ], + "Node Type": "TableLookupJoin", + "PlanNodeId": 2, + "PlanNodeType": "Connection", + "Plans": [ + { + "Node Type": "ConstantExpr", + "Operators": [ + { + "Inputs": [], + "Iterator": "[]", + "Name": "Iterator" + } + ], + "PlanNodeId": 1 + } + ], + "Table": "postgres_jointest/join0.test_plan/int8_tbl" + } + ] } - ], - "PlanNodeId": 1, - "Tables": [ - "postgres_jointest/join0.test_plan/int8_tbl" ] } ] @@ -137,12 +135,12 @@ }, { "Node Type": "Map", - "PlanNodeId": 8, + "PlanNodeId": 11, "PlanNodeType": "Connection", "Plans": [ { "Node Type": "Stage", - "PlanNodeId": 7, + "PlanNodeId": 10, "Plans": [ { "Node Type": "TableFullScan", @@ -163,7 +161,7 @@ "Table": "postgres_jointest/join0.test_plan/int8_tbl" } ], - "PlanNodeId": 6, + "PlanNodeId": 9, "Tables": [ "postgres_jointest/join0.test_plan/int8_tbl" ] @@ -213,11 +211,10 @@ "q1", "q2" ], - "scan_by": [ - "q1 (-\u221e, +\u221e)", - "q2 (-\u221e, +\u221e)" + "lookup_by": [ + "q1" ], - "type": "FullScan" + "type": "Lookup" } ] } diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join0.test_/query_6.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join0.test_/query_6.plan index 143e4b3984..dd349cc9c2 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join0.test_/query_6.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join0.test_/query_6.plan @@ -29,7 +29,7 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Limit-Filter", + "Node Type": "Limit-Filter-InnerJoin (MapJoin)-Filter-Filter", "Operators": [ { "Inputs": [ @@ -52,256 +52,232 @@ { "Inputs": [ { - "ExternalPlanNodeId": 26 + "InternalOperatorId": 2 } ], "Name": "Filter", "Predicate": "item.t1.stringu1 > item.t2.stringu2" + }, + { + "Condition": "subq1.d1 = t1.unique2", + "Inputs": [ + { + "InternalOperatorId": 3 + }, + { + "InternalOperatorId": 4 + } + ], + "Name": "InnerJoin (MapJoin)" + }, + { + "Inputs": [ + { + "ExternalPlanNodeId": 26 + } + ], + "Name": "Filter", + "Predicate": "Exist(item.subq1.d1)" + }, + { + "Inputs": [ + { + "ExternalPlanNodeId": 5 + } + ], + "Name": "Filter", + "Predicate": "Exist(item.unique2)" } ], "PlanNodeId": 27, "Plans": [ { - "Node Type": "UnionAll", - "PlanNodeId": 26, + "Node Type": "Broadcast", + "PlanNodeId": 5, "PlanNodeType": "Connection", "Plans": [ { "Node Type": "Stage", - "PlanNodeId": 25, + "PlanNodeId": 4, "Plans": [ { - "Columns": [ - "even", - "fivethous", - "four", - "hundred", - "odd", - "string4", - "stringu1", - "stringu2", - "ten", - "tenthous", - "thousand", - "twenty", - "two", - "twothousand", - "unique1", - "unique2" - ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", - "LookupKeyColumns": [ - "unique1" - ], - "Node Type": "TableLookupJoin", - "PlanNodeId": 24, + "Node Type": "UnionAll", + "PlanNodeId": 3, "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "InnerJoin (MapJoin)-Filter-Filter", + "Node Type": "Filter", "Operators": [ { - "Condition": "t1.unique2 = subq1.d1", - "Inputs": [ - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 2 - } - ], - "Name": "InnerJoin (MapJoin)" - }, - { "Inputs": [ { - "ExternalPlanNodeId": 22 + "ExternalPlanNodeId": 1 } ], "Name": "Filter", - "Predicate": "Exist(item.unique2)" - }, + "Predicate": "Exist(item.unique2) AND item.unique2 < 42" + } + ], + "PlanNodeId": 2, + "Plans": [ { - "Inputs": [ + "Node Type": "TableFullScan", + "Operators": [ { - "ExternalPlanNodeId": 19 + "Inputs": [], + "Name": "TableFullScan", + "ReadColumns": [ + "stringu1", + "unique2" + ], + "ReadRanges": [ + "unique1 (-\u221e, +\u221e)", + "unique2 (-\u221e, +\u221e)" + ], + "ReadRangesPointPrefixLen": "0", + "Scan": "Parallel", + "Table": "postgres_jointest/join0.test_plan/tenk1" } ], - "Name": "Filter", - "Predicate": "Exist(item.d1)" + "PlanNodeId": 1, + "Tables": [ + "postgres_jointest/join0.test_plan/tenk1" + ] } - ], + ] + } + ] + } + ] + } + ] + }, + { + "Node Type": "Map", + "PlanNodeId": 26, + "PlanNodeType": "Connection", + "Plans": [ + { + "Node Type": "Stage", + "PlanNodeId": 25, + "Plans": [ + { + "Node Type": "UnionAll", + "PlanNodeId": 24, + "PlanNodeType": "Connection", + "Plans": [ + { + "Node Type": "Stage", "PlanNodeId": 23, "Plans": [ { - "Node Type": "Map", + "Columns": [ + "even", + "fivethous", + "four", + "hundred", + "odd", + "string4", + "stringu1", + "stringu2", + "ten", + "tenthous", + "thousand", + "twenty", + "two", + "twothousand", + "unique1", + "unique2" + ], + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", + "LookupKeyColumns": [ + "unique1" + ], + "Node Type": "TableLookupJoin", "PlanNodeId": 22, "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Filter", + "Node Type": "LeftJoin (MapJoin)-Filter", "Operators": [ { + "Condition": "i1.f1 = subq1.v1.x2", "Inputs": [ { "ExternalPlanNodeId": 20 + }, + { + "InternalOperatorId": 1 } ], - "Name": "Filter", - "Predicate": "Exist(item.unique2) AND item.unique2 < 42" - } - ], - "PlanNodeId": 21, - "Plans": [ + "Name": "LeftJoin (MapJoin)" + }, { - "Node Type": "TableFullScan", - "Operators": [ + "Inputs": [ { - "Inputs": [], - "Name": "TableFullScan", - "ReadColumns": [ - "stringu1", - "unique2" - ], - "ReadRanges": [ - "unique1 (-\u221e, +\u221e)", - "unique2 (-\u221e, +\u221e)" - ], - "ReadRangesPointPrefixLen": "0", - "Scan": "Parallel", - "Table": "postgres_jointest/join0.test_plan/tenk1" + "ExternalPlanNodeId": 17 } ], - "PlanNodeId": 20, - "Tables": [ - "postgres_jointest/join0.test_plan/tenk1" - ] + "Name": "Filter", + "Predicate": "Exist(item.v1.x2)" } - ] - } - ] - }, - { - "Node Type": "Broadcast", - "PlanNodeId": 19, - "PlanNodeType": "Connection", - "Plans": [ - { - "Node Type": "Stage", - "PlanNodeId": 18, + ], + "PlanNodeId": 21, "Plans": [ { - "Node Type": "UnionAll", + "Node Type": "Broadcast", "PlanNodeId": 17, "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "LeftJoin (MapJoin)-Filter", - "Operators": [ - { - "Condition": "i1.f1 = subq1.v1.x2", - "Inputs": [ - { - "ExternalPlanNodeId": 15 - }, - { - "InternalOperatorId": 1 - } - ], - "Name": "LeftJoin (MapJoin)" - }, - { - "Inputs": [ - { - "ExternalPlanNodeId": 12 - } - ], - "Name": "Filter", - "Predicate": "Exist(item.v1.x2)" - } - ], + "Node Type": "Stage", "PlanNodeId": 16, "Plans": [ { - "Node Type": "Map", + "Node Type": "UnionAll", "PlanNodeId": 15, "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Stage", - "PlanNodeId": 14, - "Plans": [ + "Node Type": "LeftJoin (MapJoin)-Filter", + "Operators": [ { - "Node Type": "TableFullScan", - "Operators": [ + "Condition": "v1.x1 = v2.y2", + "Inputs": [ + { + "ExternalPlanNodeId": 13 + }, { - "Inputs": [], - "Name": "TableFullScan", - "ReadColumns": [ - "f1" - ], - "ReadRanges": [ - "f1 (-\u221e, +\u221e)" - ], - "ReadRangesPointPrefixLen": "0", - "Scan": "Parallel", - "Table": "postgres_jointest/join0.test_plan/int4_tbl" + "InternalOperatorId": 1 } ], - "PlanNodeId": 13, - "Tables": [ - "postgres_jointest/join0.test_plan/int4_tbl" - ] + "Name": "LeftJoin (MapJoin)" + }, + { + "Inputs": [ + { + "ExternalPlanNodeId": 10 + } + ], + "Name": "Filter", + "Predicate": "Exist(item.y2)" } - ] - } - ] - }, - { - "Node Type": "Broadcast", - "PlanNodeId": 12, - "PlanNodeType": "Connection", - "Plans": [ - { - "Node Type": "Stage", - "PlanNodeId": 11, + ], + "PlanNodeId": 14, "Plans": [ { - "Node Type": "UnionAll", + "Node Type": "Broadcast", "PlanNodeId": 10, "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "LeftJoin (MapJoin)-Filter", - "Operators": [ - { - "Condition": "v1.x1 = v2.y2", - "Inputs": [ - { - "ExternalPlanNodeId": 8 - }, - { - "InternalOperatorId": 1 - } - ], - "Name": "LeftJoin (MapJoin)" - }, - { - "Inputs": [ - { - "ExternalPlanNodeId": 5 - } - ], - "Name": "Filter", - "Predicate": "Exist(item.y2)" - } - ], + "Node Type": "Stage", "PlanNodeId": 9, "Plans": [ { - "Node Type": "Map", + "Node Type": "UnionAll", "PlanNodeId": 8, "PlanNodeType": "Connection", "Plans": [ @@ -316,75 +292,62 @@ "Inputs": [], "Name": "TableFullScan", "ReadColumns": [ - "x1", - "x2" + "y1", + "y2" ], "ReadRanges": [ - "x1 (-\u221e, +\u221e)", - "x2 (-\u221e, +\u221e)" + "y1 (-\u221e, +\u221e)", + "y2 (-\u221e, +\u221e)" ], "ReadRangesPointPrefixLen": "0", "Scan": "Parallel", - "Table": "postgres_jointest/join0.test_plan/ononequery1" + "Table": "postgres_jointest/join0.test_plan/ononequery2" } ], "PlanNodeId": 6, "Tables": [ - "postgres_jointest/join0.test_plan/ononequery1" + "postgres_jointest/join0.test_plan/ononequery2" ] } ] } ] - }, + } + ] + } + ] + }, + { + "Node Type": "Map", + "PlanNodeId": 13, + "PlanNodeType": "Connection", + "Plans": [ + { + "Node Type": "Stage", + "PlanNodeId": 12, + "Plans": [ { - "Node Type": "Broadcast", - "PlanNodeId": 5, - "PlanNodeType": "Connection", - "Plans": [ + "Node Type": "TableFullScan", + "Operators": [ { - "Node Type": "Stage", - "PlanNodeId": 4, - "Plans": [ - { - "Node Type": "UnionAll", - "PlanNodeId": 3, - "PlanNodeType": "Connection", - "Plans": [ - { - "Node Type": "Stage", - "PlanNodeId": 2, - "Plans": [ - { - "Node Type": "TableFullScan", - "Operators": [ - { - "Inputs": [], - "Name": "TableFullScan", - "ReadColumns": [ - "y1", - "y2" - ], - "ReadRanges": [ - "y1 (-\u221e, +\u221e)", - "y2 (-\u221e, +\u221e)" - ], - "ReadRangesPointPrefixLen": "0", - "Scan": "Parallel", - "Table": "postgres_jointest/join0.test_plan/ononequery2" - } - ], - "PlanNodeId": 1, - "Tables": [ - "postgres_jointest/join0.test_plan/ononequery2" - ] - } - ] - } - ] - } - ] + "Inputs": [], + "Name": "TableFullScan", + "ReadColumns": [ + "x1", + "x2" + ], + "ReadRanges": [ + "x1 (-\u221e, +\u221e)", + "x2 (-\u221e, +\u221e)" + ], + "ReadRangesPointPrefixLen": "0", + "Scan": "Parallel", + "Table": "postgres_jointest/join0.test_plan/ononequery1" } + ], + "PlanNodeId": 11, + "Tables": [ + "postgres_jointest/join0.test_plan/ononequery1" ] } ] @@ -398,15 +361,50 @@ ] } ] + }, + { + "Node Type": "Map", + "PlanNodeId": 20, + "PlanNodeType": "Connection", + "Plans": [ + { + "Node Type": "Stage", + "PlanNodeId": 19, + "Plans": [ + { + "Node Type": "TableFullScan", + "Operators": [ + { + "Inputs": [], + "Name": "TableFullScan", + "ReadColumns": [ + "f1" + ], + "ReadRanges": [ + "f1 (-\u221e, +\u221e)" + ], + "ReadRangesPointPrefixLen": "0", + "Scan": "Parallel", + "Table": "postgres_jointest/join0.test_plan/int4_tbl" + } + ], + "PlanNodeId": 18, + "Tables": [ + "postgres_jointest/join0.test_plan/int4_tbl" + ] + } + ] + } + ] } ] } - ] + ], + "Table": "postgres_jointest/join0.test_plan/tenk1" } ] } - ], - "Table": "postgres_jointest/join0.test_plan/tenk1" + ] } ] } diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_1.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_1.plan index 45d63232ce..146c8a41b1 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_1.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_1.plan @@ -58,9 +58,9 @@ "i", "k" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "i" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_13.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_13.plan index 0d1571bbda..03b204be82 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_13.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_13.plan @@ -75,9 +75,9 @@ "i", "k" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "i" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_14.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_14.plan index 5823923e8b..49acf35113 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_14.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_14.plan @@ -57,9 +57,9 @@ "i", "k" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "1", + "E-Size": "0", "LookupKeyColumns": [ "i" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_2.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_2.plan index 424204ffd0..5e89745a28 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_2.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_2.plan @@ -58,9 +58,9 @@ "i", "k" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "i" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_3.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_3.plan index 424204ffd0..5e89745a28 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_3.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_3.plan @@ -58,9 +58,9 @@ "i", "k" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "i" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_5.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_5.plan index 424204ffd0..5e89745a28 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_5.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_5.plan @@ -58,9 +58,9 @@ "i", "k" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "i" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_7.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_7.plan index 8779e0bb48..0b7684f328 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_7.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_7.plan @@ -58,9 +58,9 @@ "i", "k" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "i" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_8.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_8.plan index 8779e0bb48..0b7684f328 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_8.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_8.plan @@ -58,9 +58,9 @@ "i", "k" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "i" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_1.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_1.plan index 2b23878c57..eeeb71e16f 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_1.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_1.plan @@ -62,7 +62,7 @@ "ExternalPlanNodeId": 19 }, { - "ExternalPlanNodeId": 14 + "ExternalPlanNodeId": 5 } ], "Name": "FullJoin (JoinDict)" @@ -75,16 +75,69 @@ "name" ], "Node Type": "HashShuffle", - "PlanNodeId": 14, + "PlanNodeId": 5, "PlanNodeType": "Connection", "Plans": [ { "Node Type": "Stage", - "PlanNodeId": 13, + "PlanNodeId": 4, "Plans": [ { "Node Type": "UnionAll", - "PlanNodeId": 12, + "PlanNodeId": 3, + "PlanNodeType": "Connection", + "Plans": [ + { + "Node Type": "Stage", + "PlanNodeId": 2, + "Plans": [ + { + "Node Type": "TableFullScan", + "Operators": [ + { + "Inputs": [], + "Name": "TableFullScan", + "ReadColumns": [ + "n", + "name" + ], + "ReadRanges": [ + "name (-\u221e, +\u221e)", + "n (-\u221e, +\u221e)" + ], + "ReadRangesPointPrefixLen": "0", + "Scan": "Parallel", + "Table": "postgres_jointest/join2.test_plan/t3" + } + ], + "PlanNodeId": 1, + "Tables": [ + "postgres_jointest/join2.test_plan/t3" + ] + } + ] + } + ] + } + ] + } + ] + }, + { + "KeyColumns": [ + "name" + ], + "Node Type": "HashShuffle", + "PlanNodeId": 19, + "PlanNodeType": "Connection", + "Plans": [ + { + "Node Type": "Stage", + "PlanNodeId": 18, + "Plans": [ + { + "Node Type": "UnionAll", + "PlanNodeId": 17, "PlanNodeType": "Connection", "Plans": [ { @@ -93,16 +146,16 @@ { "Inputs": [ { - "ExternalPlanNodeId": 10 + "ExternalPlanNodeId": 15 }, { - "ExternalPlanNodeId": 5 + "ExternalPlanNodeId": 10 } ], "Name": "FullJoin (JoinDict)" } ], - "PlanNodeId": 11, + "PlanNodeId": 16, "Plans": [ { "KeyColumns": [ @@ -141,12 +194,12 @@ ], "ReadRangesPointPrefixLen": "0", "Scan": "Parallel", - "Table": "postgres_jointest/join2.test_plan/t1" + "Table": "postgres_jointest/join2.test_plan/t2" } ], "PlanNodeId": 6, "Tables": [ - "postgres_jointest/join2.test_plan/t1" + "postgres_jointest/join2.test_plan/t2" ] } ] @@ -162,21 +215,21 @@ "name" ], "Node Type": "HashShuffle", - "PlanNodeId": 5, + "PlanNodeId": 15, "PlanNodeType": "Connection", "Plans": [ { "Node Type": "Stage", - "PlanNodeId": 4, + "PlanNodeId": 14, "Plans": [ { "Node Type": "UnionAll", - "PlanNodeId": 3, + "PlanNodeId": 13, "PlanNodeType": "Connection", "Plans": [ { "Node Type": "Stage", - "PlanNodeId": 2, + "PlanNodeId": 12, "Plans": [ { "Node Type": "TableFullScan", @@ -194,12 +247,12 @@ ], "ReadRangesPointPrefixLen": "0", "Scan": "Parallel", - "Table": "postgres_jointest/join2.test_plan/t2" + "Table": "postgres_jointest/join2.test_plan/t1" } ], - "PlanNodeId": 1, + "PlanNodeId": 11, "Tables": [ - "postgres_jointest/join2.test_plan/t2" + "postgres_jointest/join2.test_plan/t1" ] } ] @@ -217,59 +270,6 @@ ] } ] - }, - { - "KeyColumns": [ - "name" - ], - "Node Type": "HashShuffle", - "PlanNodeId": 19, - "PlanNodeType": "Connection", - "Plans": [ - { - "Node Type": "Stage", - "PlanNodeId": 18, - "Plans": [ - { - "Node Type": "UnionAll", - "PlanNodeId": 17, - "PlanNodeType": "Connection", - "Plans": [ - { - "Node Type": "Stage", - "PlanNodeId": 16, - "Plans": [ - { - "Node Type": "TableFullScan", - "Operators": [ - { - "Inputs": [], - "Name": "TableFullScan", - "ReadColumns": [ - "n", - "name" - ], - "ReadRanges": [ - "name (-\u221e, +\u221e)", - "n (-\u221e, +\u221e)" - ], - "ReadRangesPointPrefixLen": "0", - "Scan": "Parallel", - "Table": "postgres_jointest/join2.test_plan/t3" - } - ], - "PlanNodeId": 15, - "Tables": [ - "postgres_jointest/join2.test_plan/t3" - ] - } - ] - } - ] - } - ] - } - ] } ] } diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_10.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_10.plan index f919467a04..e68167a6da 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_10.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_10.plan @@ -138,9 +138,9 @@ "y1", "y2" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "y1" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_11.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_11.plan index a61d0a619e..23d554bd9e 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_11.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_11.plan @@ -174,9 +174,9 @@ "y1", "y2" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "y1" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_12.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_12.plan index a51768f6ce..4e3691af54 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_12.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_12.plan @@ -138,9 +138,9 @@ "y1", "y2" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "y1" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_2.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_2.plan index b332512c7a..c44ae4876e 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_2.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_2.plan @@ -58,9 +58,9 @@ "n", "name" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "name" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_3.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_3.plan index b332512c7a..c44ae4876e 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_3.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_3.plan @@ -58,9 +58,9 @@ "n", "name" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "name" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_7.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_7.plan index 5487a78ee2..92841f40ae 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_7.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_7.plan @@ -58,9 +58,9 @@ "y1", "y2" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "y1" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_8.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_8.plan index d064b26895..faddfd87c8 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_8.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_8.plan @@ -76,9 +76,9 @@ "y1", "y2" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "y1" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_9.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_9.plan index a51768f6ce..4e3691af54 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_9.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_9.plan @@ -138,9 +138,9 @@ "y1", "y2" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "y1" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_2.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_2.plan index c47d2f757e..59c2ef9836 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_2.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_2.plan @@ -57,9 +57,9 @@ "Columns": [ "k" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "k" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_4.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_4.plan index 75b8111d8e..92384ba828 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_4.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_4.plan @@ -56,9 +56,9 @@ "Columns": [ "k" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "k" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_5.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_5.plan index 29e3d9c1bc..c219b568b3 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_5.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_5.plan @@ -57,9 +57,9 @@ "k", "pd" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "k" ], @@ -84,9 +84,9 @@ "Columns": [ "k" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "k" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_6.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_6.plan index 0cf5446b69..9e7b6ec48e 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_6.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_6.plan @@ -71,9 +71,9 @@ "Columns": [ "id" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "id" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_7.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_7.plan index 124f92c63f..9763ab4b6c 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_7.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_7.plan @@ -68,9 +68,9 @@ "Columns": [ "id" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "id" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_13.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_13.plan index 02e8e5be4e..1f5aa48267 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_13.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_13.plan @@ -58,9 +58,9 @@ "a", "b" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "a" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_5.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_5.plan index b75dbc620d..7051ae29e0 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_5.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_5.plan @@ -57,9 +57,9 @@ "x", "y" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "x", "y" diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_6.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_6.plan index 7f6611e3e1..8fcf8d9196 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_6.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_6.plan @@ -5,7 +5,7 @@ "Plans": [ { "Node Type": "ResultSet", - "PlanNodeId": 13, + "PlanNodeId": 16, "PlanNodeType": "ResultSet", "Plans": [ { @@ -14,55 +14,67 @@ { "Inputs": [ { - "ExternalPlanNodeId": 11 + "ExternalPlanNodeId": 14 } ], "Limit": "1001", "Name": "Limit" } ], - "PlanNodeId": 12, + "PlanNodeId": 15, "Plans": [ { "Node Type": "UnionAll", - "PlanNodeId": 11, + "PlanNodeId": 14, "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Limit", + "Node Type": "Limit-LeftJoin (MapJoin)-Filter", "Operators": [ { "Inputs": [ { - "ExternalPlanNodeId": 9 + "InternalOperatorId": 1 } ], "Limit": "1001", "Name": "Limit" + }, + { + "Condition": "zt2.f2 = zt3.f3", + "Inputs": [ + { + "ExternalPlanNodeId": 12 + }, + { + "InternalOperatorId": 2 + } + ], + "Name": "LeftJoin (MapJoin)" + }, + { + "Inputs": [ + { + "ExternalPlanNodeId": 9 + } + ], + "Name": "Filter", + "Predicate": "Exist(item.zt3.f3)" } ], - "PlanNodeId": 10, + "PlanNodeId": 13, "Plans": [ { - "Node Type": "UnionAll", + "Node Type": "Broadcast", "PlanNodeId": 9, "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Collect", + "Node Type": "Stage", "PlanNodeId": 8, "Plans": [ { - "Columns": [ - "f1" - ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", - "LookupKeyColumns": [ - "f1" - ], - "Node Type": "TableLookupJoin", + "Node Type": "UnionAll", "PlanNodeId": 7, "PlanNodeType": "Connection", "Plans": [ @@ -81,13 +93,13 @@ "Plans": [ { "Columns": [ - "f3" + "f1" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ - "f3" + "f1" ], "Node Type": "TableLookupJoin", "PlanNodeId": 3, @@ -98,30 +110,31 @@ "PlanNodeId": 2, "Plans": [ { - "Node Type": "TablePointLookup", + "Node Type": "TableFullScan", "Operators": [ { "Inputs": [], - "Name": "TablePointLookup", + "Name": "TableFullScan", "ReadColumns": [ - "f2" + "f3" ], - "ReadRange": [ - "f2 (53)" + "ReadRanges": [ + "f3 (-\u221e, +\u221e)" ], + "ReadRangesPointPrefixLen": "0", "Scan": "Parallel", - "Table": "postgres_jointest/join4.test_plan/zt2" + "Table": "postgres_jointest/join4.test_plan/zt3" } ], "PlanNodeId": 1, "Tables": [ - "postgres_jointest/join4.test_plan/zt2" + "postgres_jointest/join4.test_plan/zt3" ] } ] } ], - "Table": "postgres_jointest/join4.test_plan/zt3" + "Table": "postgres_jointest/join4.test_plan/zt1" } ] } @@ -129,8 +142,41 @@ } ] } + ] + } + ] + } + ] + }, + { + "Node Type": "Map", + "PlanNodeId": 12, + "PlanNodeType": "Connection", + "Plans": [ + { + "Node Type": "Stage", + "PlanNodeId": 11, + "Plans": [ + { + "Node Type": "TablePointLookup", + "Operators": [ + { + "Inputs": [], + "Name": "TablePointLookup", + "ReadColumns": [ + "f2" + ], + "ReadRange": [ + "f2 (53)" + ], + "Scan": "Parallel", + "Table": "postgres_jointest/join4.test_plan/zt2" + } ], - "Table": "postgres_jointest/join4.test_plan/zt1" + "PlanNodeId": 10, + "Tables": [ + "postgres_jointest/join4.test_plan/zt2" + ] } ] } @@ -189,10 +235,10 @@ "columns": [ "f3" ], - "lookup_by": [ - "f3" + "scan_by": [ + "f3 (-\u221e, +\u221e)" ], - "type": "Lookup" + "type": "FullScan" } ] } diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_7.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_7.plan index cc69f3e676..776d05a40c 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_7.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_7.plan @@ -75,9 +75,9 @@ "f1", "f2" ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", + "E-Cost": "0", + "E-Rows": "0", + "E-Size": "0", "LookupKeyColumns": [ "f1" ], diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-select.test_/query_12.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-select.test_/query_12.plan index 4b9b282f98..edb2b1f04c 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-select.test_/query_12.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-select.test_/query_12.plan @@ -41,7 +41,7 @@ "Name": "Limit" }, { - "Condition": "onek.stringu1,onek.unique1 = v.i,v.j", + "Condition": "onek.stringu1,onek.unique1 = v.j,v.i", "Inputs": [ { "InternalOperatorId": 2 |