diff options
| author | Denis Khalikov <deniskhalikov@ydb.tech> | 2025-04-30 11:16:07 +0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2025-04-30 11:16:07 +0300 |
| commit | 9c0e5384e3da52b2c10b167f731c07fce1beae9c (patch) | |
| tree | 793bd7797ff32e7eabb6b630d6ccc7b2a4ba6689 | |
| parent | fdc10e3013349760e1dee76f0be41d6d8b6cb985 (diff) | |
| download | ydb-9c0e5384e3da52b2c10b167f731c07fce1beae9c.tar.gz | |
Enable YQL core opt flags from kqp config (#17704)
15 files changed, 930 insertions, 646 deletions
diff --git a/ydb/core/kqp/compile_service/kqp_compile_actor.cpp b/ydb/core/kqp/compile_service/kqp_compile_actor.cpp index 733fda15c25..624ed9c3545 100644 --- a/ydb/core/kqp/compile_service/kqp_compile_actor.cpp +++ b/ydb/core/kqp/compile_service/kqp_compile_actor.cpp @@ -658,6 +658,14 @@ void ApplyServiceConfig(TKikimrConfiguration& kqpConfig, const TTableServiceConf if (const auto limit = serviceConfig.GetResourceManager().GetMkqlHeavyProgramMemoryLimit()) { kqpConfig._KqpYqlCombinerMemoryLimit = std::max(1_GB, limit - (limit >> 2U)); } + + kqpConfig.FilterPushdownOverJoinOptionalSide = serviceConfig.GetFilterPushdownOverJoinOptionalSide(); + if (serviceConfig.GetFuseEquiJoinsInputMultiLabels()) + kqpConfig.YqlCoreOptimizerFlags.insert("fuseequijoinsinputmultilabels"); + if (serviceConfig.GetPullUpFlatMapOverJoinMultipleLabels()) + kqpConfig.YqlCoreOptimizerFlags.insert("pullupflatmapoverjoinmultiplelabels"); + if (serviceConfig.GetEqualityFilterOverJoin()) + kqpConfig.YqlCoreOptimizerFlags.insert("equalityfilteroverjoin"); } IActor* CreateKqpCompileActor(const TActorId& owner, const TKqpSettings::TConstPtr& kqpSettings, diff --git a/ydb/core/kqp/host/kqp_host.cpp b/ydb/core/kqp/host/kqp_host.cpp index f3dcf8337f4..6ab3645dada 100644 --- a/ydb/core/kqp/host/kqp_host.cpp +++ b/ydb/core/kqp/host/kqp_host.cpp @@ -1929,6 +1929,9 @@ private: TypesCtx->AddDataSource(providerNames, kikimrDataSource); TypesCtx->AddDataSink(providerNames, kikimrDataSink); + TypesCtx->FilterPushdownOverJoinOptionalSide = SessionCtx->ConfigPtr()->FilterPushdownOverJoinOptionalSide; + const auto &yqlCoreOptFlags = SessionCtx->ConfigPtr()->YqlCoreOptimizerFlags; + TypesCtx->OptimizerFlags.insert(yqlCoreOptFlags.begin(), yqlCoreOptFlags.end()); bool addExternalDataSources = queryType == EKikimrQueryType::Script || queryType == EKikimrQueryType::Query || (queryType == EKikimrQueryType::YqlScript || queryType == EKikimrQueryType::YqlScriptStreaming) && AppData()->FeatureFlags.GetEnableExternalDataSources(); diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.h b/ydb/core/kqp/provider/yql_kikimr_settings.h index e88d6f3143d..018b8733198 100644 --- a/ydb/core/kqp/provider/yql_kikimr_settings.h +++ b/ydb/core/kqp/provider/yql_kikimr_settings.h @@ -183,6 +183,8 @@ struct TKikimrConfiguration : public TKikimrSettings, public NCommon::TSettingDi bool EnableSnapshotIsolationRW = false; bool AllowMultiBroadcasts = false; bool DefaultEnableShuffleElimination = false; + bool FilterPushdownOverJoinOptionalSide = false; + THashSet<TString> YqlCoreOptimizerFlags; void SetDefaultEnabledSpillingNodes(const TString& node); ui64 GetEnabledSpillingNodes() const; diff --git a/ydb/core/kqp/ut/join/data/join_order/lookupbug.json b/ydb/core/kqp/ut/join/data/join_order/lookupbug.json index b32ce9858cd..d9075507c0a 100644 --- a/ydb/core/kqp/ut/join/data/join_order/lookupbug.json +++ b/ydb/core/kqp/ut/join/data/join_order/lookupbug.json @@ -3,32 +3,32 @@ "args": [ { - "op_name":"LeftJoin (MapJoin)", + "op_name":"InnerJoin (MapJoin)", "args": [ { - "op_name":"LeftJoin (MapJoin)", + "op_name":"InnerJoin (MapJoin)", "args": [ { "op_name":"TableFullScan", - "table":"quotas_browsers_relation" + "table":"browsers" }, { - "op_name":"TableLookup", - "table":"browsers" + "op_name":"TableFullScan", + "table":"quotas_browsers_relation" } ] }, { - "op_name":"TableLookup", - "table":"browser_groups" + "op_name":"TablePointLookup", + "table":"quota" } ] }, { - "op_name":"TableFullScan", - "table":"quota" + "op_name":"TableLookup", + "table":"browser_groups" } ] } diff --git a/ydb/core/kqp/ut/join/kqp_index_lookup_join_ut.cpp b/ydb/core/kqp/ut/join/kqp_index_lookup_join_ut.cpp index c953bec132e..420e639d827 100644 --- a/ydb/core/kqp/ut/join/kqp_index_lookup_join_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_index_lookup_join_ut.cpp @@ -1,7 +1,6 @@ #include <ydb/core/kqp/ut/common/kqp_ut_common.h> #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/proto/accessor.h> - #include <fmt/format.h> namespace NKikimr { @@ -101,7 +100,7 @@ void PrepareTables(TSession session) { Y_UNIT_TEST_SUITE(KqpIndexLookupJoin) { -void Test(const TString& query, const TString& answer, size_t rightTableReads, bool useStreamLookup = false) { +void Test(const TString& query, const TString& answer, size_t rightTableReads, bool useStreamLookup = false, size_t leftTableReads = 7) { NKikimrConfig::TAppConfig appConfig; appConfig.MutableTableServiceConfig()->SetEnableKqpDataQueryStreamIdxLookupJoin(useStreamLookup); @@ -125,9 +124,10 @@ void Test(const TString& query, const TString& answer, size_t rightTableReads, b UNIT_ASSERT_VALUES_EQUAL(stats.query_phases().size(), 1); UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(0).table_access().size(), 2); + for (const auto& tableStat : stats.query_phases(0).table_access()) { if (tableStat.name() == "/Root/Left") { - UNIT_ASSERT_VALUES_EQUAL(tableStat.reads().rows(), 7); + UNIT_ASSERT_VALUES_EQUAL(tableStat.reads().rows(), leftTableReads); } else { UNIT_ASSERT_VALUES_EQUAL(tableStat.name(), "/Root/Right"); UNIT_ASSERT_VALUES_EQUAL(tableStat.reads().rows(), rightTableReads); @@ -519,7 +519,7 @@ Y_UNIT_TEST_TWIN(LeftJoinRightNullFilter, StreamLookup) { [["Value3"];#]; [["Value6"];#]; [["Value7"];#] - ])", 4, StreamLookup); + ])", 8, StreamLookup, 14); } Y_UNIT_TEST_TWIN(LeftJoinSkipNullFilter, StreamLookup) { diff --git a/ydb/core/protos/table_service_config.proto b/ydb/core/protos/table_service_config.proto index fb1dd9f7aff..2739d5c77a5 100644 --- a/ydb/core/protos/table_service_config.proto +++ b/ydb/core/protos/table_service_config.proto @@ -384,4 +384,9 @@ message TTableServiceConfig { optional bool EnableFoldUdfs = 82 [ default = true ]; + // YQL core optimizer flags. + optional bool FilterPushdownOverJoinOptionalSide = 83 [ default = true ]; + optional bool FuseEquiJoinsInputMultiLabels = 84 [ default = true ]; + optional bool PullUpFlatMapOverJoinMultipleLabels = 85 [ default = true ]; + optional bool EqualityFilterOverJoin = 86 [ default = false ]; }; diff --git a/ydb/library/yql/dq/opt/dq_opt_join.cpp b/ydb/library/yql/dq/opt/dq_opt_join.cpp index 9d30768fce9..2a3ff760dc4 100644 --- a/ydb/library/yql/dq/opt/dq_opt_join.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_join.cpp @@ -15,17 +15,17 @@ using namespace NYql::NNodes; namespace { struct TJoinInputDesc { - TJoinInputDesc(TMaybe<TStringBuf> label, const TExprBase& input, + TJoinInputDesc(TMaybe<THashSet<TStringBuf>> labels, const TExprBase& input, TSet<std::pair<TStringBuf, TStringBuf>>&& keys) - : Label(label) + : Labels(labels) , Input(input) , Keys(std::move(keys)) {} bool IsRealTable() const { - return Label.Defined(); + return Labels.Defined(); } - TMaybe<TStringBuf> Label; // defined for real table input only, empty otherwise + TMaybe<THashSet<TStringBuf>> Labels; // defined for real table input only, empty otherwise TExprBase Input; TSet<std::pair<TStringBuf, TStringBuf>> Keys; // set of (label, column_name) pairs in this input }; @@ -116,6 +116,14 @@ TExprBase BuildDqJoinInput(TExprContext& ctx, TPositionHandle pos, const TExprBa return partition; } +TExprNode::TPtr CreateLabelList(const THashSet<TStringBuf>& labels, const TPositionHandle& position, TExprContext& ctx) { + TExprNode::TListType newKeys; + for (const auto& label : labels) { + newKeys.push_back(ctx.NewAtom(position, label)); + } + return ctx.NewList(position, std::move(newKeys)); +} + TMaybe<TJoinInputDesc> BuildDqJoin( const TCoEquiJoinTuple& joinTuple, const THashMap<TStringBuf, TJoinInputDesc>& inputs, @@ -129,9 +137,12 @@ TMaybe<TJoinInputDesc> BuildDqJoin( { TMaybe<TJoinInputDesc> left; TVector<TString> lhsLabels; + TStringBuf leftLabel; + TStringBuf rightLabel; if (joinTuple.LeftScope().Maybe<TCoAtom>()) { lhsLabels.push_back(joinTuple.LeftScope().Cast<TCoAtom>().StringValue()); left = inputs.at(joinTuple.LeftScope().Cast<TCoAtom>().Value()); + leftLabel = joinTuple.LeftScope().Cast<TCoAtom>().Value(); YQL_ENSURE(left, "unknown scope " << joinTuple.LeftScope().Cast<TCoAtom>().Value()); } else { left = BuildDqJoin(joinTuple.LeftScope().Cast<TCoEquiJoinTuple>(), inputs, mode, ctx, typeCtx, lhsLabels, hints, useCBO); @@ -145,6 +156,7 @@ TMaybe<TJoinInputDesc> BuildDqJoin( if (joinTuple.RightScope().Maybe<TCoAtom>()) { rhsLabels.push_back(joinTuple.RightScope().Cast<TCoAtom>().StringValue()); right = inputs.at(joinTuple.RightScope().Cast<TCoAtom>().Value()); + rightLabel = joinTuple.RightScope().Cast<TCoAtom>().Value(); YQL_ENSURE(right, "unknown scope " << joinTuple.RightScope().Cast<TCoAtom>().Value()); } else { right = BuildDqJoin(joinTuple.RightScope().Cast<TCoEquiJoinTuple>(), inputs, mode, ctx, typeCtx, rhsLabels, hints, useCBO); @@ -187,12 +199,13 @@ TMaybe<TJoinInputDesc> BuildDqJoin( resultKeys.insert(right->Keys.begin(), right->Keys.end()); } - auto leftTableLabel = left->IsRealTable() - ? BuildAtom(*left->Label, left->Input.Pos(), ctx).Ptr() - : Build<TCoVoid>(ctx, left->Input.Pos()).Done().Ptr(); - auto rightTableLabel = right->IsRealTable() - ? BuildAtom(*right->Label, right->Input.Pos(), ctx).Ptr() - : Build<TCoVoid>(ctx, right->Input.Pos()).Done().Ptr(); + auto leftTableLabel = left->IsRealTable() ? (left->Labels->size() > 1 ? CreateLabelList(*(left->Labels), left->Input.Pos(), ctx) + : BuildAtom(leftLabel, left->Input.Pos(), ctx).Ptr()) + : Build<TCoVoid>(ctx, left->Input.Pos()).Done().Ptr(); + + auto rightTableLabel = right->IsRealTable() ? (right->Labels->size() > 1 ? CreateLabelList(*(right->Labels), right->Input.Pos(), ctx) + : BuildAtom(rightLabel, right->Input.Pos(), ctx).Ptr()) + : Build<TCoVoid>(ctx, right->Input.Pos()).Done().Ptr(); size_t joinKeysCount = joinTuple.LeftKeys().Size() / 2; TVector<TCoAtom> leftJoinKeys; @@ -353,21 +366,37 @@ TMaybe<TJoinInputDesc> BuildDqJoin( } TMaybe<TJoinInputDesc> PrepareJoinInput(const TCoEquiJoinInput& input) { - if (!input.Scope().Maybe<TCoAtom>()) { - YQL_CLOG(TRACE, CoreDq) << "EquiJoin input scope is not an Atom: " << input.Scope().Ref().Content(); - return {}; + THashSet<TStringBuf> labels; + if (input.Scope().Maybe<TCoAtom>()) { + labels.insert(input.Scope().Cast<TCoAtom>().Value()); + } else { + auto list = input.Scope().Cast<TCoAtomList>(); + for (auto atomLabel : list) { + labels.insert(atomLabel.Value()); + } } - auto scope = input.Scope().Cast<TCoAtom>().Value(); auto listType = input.List().Ref().GetTypeAnn()->Cast<TListExprType>(); auto resultStructType = listType->GetItemType()->Cast<TStructExprType>(); TSet<std::pair<TStringBuf, TStringBuf>> keys; for (auto member : resultStructType->GetItems()) { - keys.emplace(scope, member->GetName()); + if (input.Scope().Maybe<TCoAtom>()) { + keys.emplace(input.Scope().Cast<TCoAtom>().Value(), member->GetName()); + } else { + auto fullMemberName = member->GetName(); + if (fullMemberName.find(".") != TString::npos) { + TStringBuf table; + TStringBuf column; + SplitTableName(fullMemberName, table, column); + keys.emplace(table, column); + } else { + return {}; + } + } } - return TJoinInputDesc(scope, input.List(), std::move(keys)); + return TJoinInputDesc(labels, input.List(), std::move(keys)); } TStringBuf RotateRightJoinType(TStringBuf joinType) { @@ -396,13 +425,13 @@ std::pair<TVector<TCoAtom>, TVector<TCoAtom>> GetJoinKeys(const TDqJoin& join, T auto rightLabel = keyTuple.RightLabel().Value(); auto leftKey = Build<TCoAtom>(ctx, join.Pos()) - .Value(join.LeftLabel().Maybe<TCoAtom>() || keyTuple.LeftColumn().Value().starts_with("_yql_dq_key_left_") + .Value((join.LeftLabel().Maybe<TCoAtom>() || keyTuple.LeftColumn().Value().starts_with("_yql_dq_key_left_")) && !join.LeftLabel().Maybe<TCoAtomList>() ? keyTuple.LeftColumn().StringValue() : FullColumnName(leftLabel, keyTuple.LeftColumn().Value())) .Done(); auto rightKey = Build<TCoAtom>(ctx, join.Pos()) - .Value(join.RightLabel().Maybe<TCoAtom>() || keyTuple.RightColumn().Value().starts_with("_yql_dq_key_right_") + .Value((join.RightLabel().Maybe<TCoAtom>() || keyTuple.RightColumn().Value().starts_with("_yql_dq_key_right_")) && !join.RightLabel().Maybe<TCoAtomList>() ? keyTuple.RightColumn().StringValue() : FullColumnName(rightLabel, keyTuple.RightColumn().Value())) .Done(); @@ -414,7 +443,6 @@ std::pair<TVector<TCoAtom>, TVector<TCoAtom>> GetJoinKeys(const TDqJoin& join, T return std::make_pair(std::move(leftJoinKeys), std::move(rightJoinKeys)); } - TDqJoinBase DqMakePhyMapJoin(const TDqJoin& join, const TExprBase& leftInput, const TExprBase& rightInput, TExprContext& ctx, bool useGraceCore) { @@ -521,7 +549,9 @@ TExprBase DqRewriteEquiJoin( THashMap<TStringBuf, TJoinInputDesc> inputs; for (size_t i = 0; i < equiJoin.ArgCount() - 2; ++i) { if (auto input = PrepareJoinInput(equiJoin.Arg(i).Cast<TCoEquiJoinInput>())) { - inputs.emplace(*input->Label, std::move(*input)); + for (auto label : *(input->Labels)) { + inputs.emplace(label, *input); + } } else { return node; } diff --git a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp index b90565682b2..49c77c9ddaf 100644 --- a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp @@ -30,13 +30,13 @@ bool DqCollectJoinRelationsWithStats( auto stats = typesCtx.GetStats(joinArg.Raw()); - if (!stats) { - YQL_CLOG(TRACE, CoreDq) << "Didn't find statistics for scope " << input.Scope().Cast<TCoAtom>().StringValue() << "\n"; + auto scope = input.Scope(); + if (!scope.Maybe<TCoAtom>()){ return false; } - auto scope = input.Scope(); - if (!scope.Maybe<TCoAtom>()){ + if (!stats) { + YQL_CLOG(TRACE, CoreDq) << "Didn't find statistics for scope " << input.Scope().Cast<TCoAtom>().StringValue() << "\n"; return false; } diff --git a/ydb/library/yql/dq/type_ann/dq_type_ann.cpp b/ydb/library/yql/dq/type_ann/dq_type_ann.cpp index f216448e7c1..5214ef7643f 100644 --- a/ydb/library/yql/dq/type_ann/dq_type_ann.cpp +++ b/ydb/library/yql/dq/type_ann/dq_type_ann.cpp @@ -258,7 +258,7 @@ TStatus AnnotateStage(const TExprNode::TPtr& stage, TExprContext& ctx) { } THashMap<TStringBuf, THashMap<TStringBuf, const TTypeAnnotationNode*>> -ParseJoinInputType(const TStructExprType& rowType, TStringBuf tableLabel, TExprContext& ctx, bool optional) { +ParseJoinInputType(const TStructExprType& rowType, const THashSet<TStringBuf>& tableLabels, TExprContext& ctx, bool optional) { THashMap<TStringBuf, THashMap<TStringBuf, const TTypeAnnotationNode*>> result; for (auto member : rowType.GetItems()) { TStringBuf label, column; @@ -268,7 +268,7 @@ ParseJoinInputType(const TStructExprType& rowType, TStringBuf tableLabel, TExprC column = member->GetName(); } const bool isSystemKeyColumn = column.starts_with("_yql_dq_key_"); - if (label.empty() && tableLabel.empty() && !isSystemKeyColumn) { + if (label.empty() && (tableLabels.size() == 1 && tableLabels.begin()->empty()) && !isSystemKeyColumn) { ctx.AddError(TIssue(TStringBuilder() << "Invalid join input type " << FormatType(&rowType))); result.clear(); return result; @@ -277,10 +277,17 @@ ParseJoinInputType(const TStructExprType& rowType, TStringBuf tableLabel, TExprC if (optional && !memberType->IsOptionalOrNull()) { memberType = ctx.MakeType<TOptionalExprType>(memberType); } - if (!tableLabel.empty()) { - result[tableLabel][member->GetName()] = memberType; - } else { + if (tableLabels.size() > 1) { + YQL_ENSURE(label); + YQL_ENSURE(column); result[label][column] = memberType; + } else { + YQL_ENSURE(tableLabels.size() == 1); + if (!(tableLabels.begin())->empty()) { + result[*(tableLabels.begin())][member->GetName()] = memberType; + } else { + result[label][column] = memberType; + } } } return result; @@ -288,12 +295,12 @@ ParseJoinInputType(const TStructExprType& rowType, TStringBuf tableLabel, TExprC template <bool IsMapJoin> const TStructExprType* GetDqJoinResultType(TPositionHandle pos, const TStructExprType& leftRowType, - const TStringBuf& leftLabel, const TStructExprType& rightRowType, const TStringBuf& rightLabel, + const THashSet<TStringBuf>& leftLabels, const TStructExprType& rightRowType, const THashSet<TStringBuf>& rightLabels, const TStringBuf& joinType, const TDqJoinKeyTupleList& joinKeys, TExprContext& ctx) { // check left bool isLeftOptional = IsLeftJoinSideOptional(joinType); - auto leftType = ParseJoinInputType(leftRowType, leftLabel, ctx, isLeftOptional); + auto leftType = ParseJoinInputType(leftRowType, leftLabels, ctx, isLeftOptional); if (leftType.empty() && joinType != "Cross") { TStringStream str; str << "Cannot parse left join input type: "; leftRowType.Out(str); @@ -303,7 +310,7 @@ const TStructExprType* GetDqJoinResultType(TPositionHandle pos, const TStructExp // check right bool isRightOptional = IsRightJoinSideOptional(joinType); - auto rightType = ParseJoinInputType(rightRowType, rightLabel, ctx, isRightOptional); + auto rightType = ParseJoinInputType(rightRowType, rightLabels, ctx, isRightOptional); if (rightType.empty() && joinType != "Cross") { TStringStream str; str << "Cannot parse right join input type: "; rightRowType.Out(str); @@ -331,11 +338,11 @@ const TStructExprType* GetDqJoinResultType(TPositionHandle pos, const TStructExp auto rightKeyLabel = key.RightLabel().Value(); auto rightKeyColumn = key.RightColumn().Value(); - if (leftLabel && leftLabel != leftKeyLabel) { + if ((leftLabels.size() && !leftLabels.begin()->empty()) && !leftLabels.contains(leftKeyLabel)) { ctx.AddError(TIssue(ctx.GetPosition(pos), "different labels for left table")); return nullptr; } - if (rightLabel && rightLabel != rightKeyLabel) { + if ((rightLabels.size() && !rightLabels.begin()->empty()) && !rightLabels.contains(rightKeyLabel)) { ctx.AddError(TIssue(ctx.GetPosition(pos), "different labels for right table")); return nullptr; } @@ -402,14 +409,26 @@ const TStructExprType* GetDqJoinResultType(const TExprNode::TPtr& input, bool st } if (!input->Child(TDqJoin::idx_LeftLabel)->IsCallable("Void")) { - if (!EnsureAtom(*input->Child(TDqJoin::idx_LeftLabel), ctx)) { - return nullptr; + if ((input->Child(TDqJoin::idx_LeftLabel)->IsAtom())) { + if (!EnsureAtom(*input->Child(TDqJoin::idx_LeftLabel), ctx)) { + return nullptr; + } + } else { + if (!EnsureTupleOfAtoms(*input->Child(TDqJoin::idx_LeftLabel), ctx)) { + return nullptr; + } } } if (!input->Child(TDqJoin::idx_RightLabel)->IsCallable("Void")) { - if (!EnsureAtom(*input->Child(TDqJoin::idx_RightLabel), ctx)) { - return nullptr; + if ((input->Child(TDqJoin::idx_RightLabel)->IsAtom())) { + if (!EnsureAtom(*input->Child(TDqJoin::idx_RightLabel), ctx)) { + return nullptr; + } + } else { + if (!EnsureTupleOfAtoms(*input->Child(TDqJoin::idx_RightLabel), ctx)) { + return nullptr; + } } } @@ -459,18 +478,32 @@ const TStructExprType* GetDqJoinResultType(const TExprNode::TPtr& input, bool st return nullptr; } auto leftStructType = leftInputItemType.Cast<TStructExprType>(); - auto leftTableLabel = join.LeftLabel().Maybe<TCoAtom>() - ? join.LeftLabel().Cast<TCoAtom>().Value() - : TStringBuf(""); + THashSet<TStringBuf> leftTableLabels; + if (join.LeftLabel().Maybe<TCoAtom>()) { + leftTableLabels.emplace(join.LeftLabel().Cast<TCoAtom>().Value()); + } else if (join.LeftLabel().Maybe<TCoAtomList>()) { + for (auto label : join.LeftLabel().Cast<TCoAtomList>()) { + leftTableLabels.emplace(label.Value()); + } + } else { + leftTableLabels.emplace(""); + } const auto& rightInputItemType = GetSeqItemType(*rightInputType); if (!EnsureStructType(join.Pos(), rightInputItemType, ctx)) { return nullptr; } auto rightStructType = rightInputItemType.Cast<TStructExprType>(); - auto rightTableLabel = join.RightLabel().Maybe<TCoAtom>() - ? join.RightLabel().Cast<TCoAtom>().Value() - : TStringBuf(""); + THashSet<TStringBuf> rightTableLabels; + if (join.RightLabel().Maybe<TCoAtom>()) { + rightTableLabels.emplace(join.RightLabel().Cast<TCoAtom>().Value()); + } else if (join.RightLabel().Maybe<TCoAtomList>()) { + for (auto label : join.RightLabel().Cast<TCoAtomList>()) { + rightTableLabels.emplace(label.Value()); + } + } else { + rightTableLabels.emplace(""); + } if (input->ChildrenSize() > TDqJoin::idx_JoinAlgoOptions) { const auto& joinAlgo = *input->Child(TDqJoin::idx_JoinAlgo); @@ -511,9 +544,9 @@ const TStructExprType* GetDqJoinResultType(const TExprNode::TPtr& input, bool st } } - return GetDqJoinResultType<IsMapJoin>(join.Pos(), *leftStructType, leftTableLabel, *rightStructType, - rightTableLabel, join.JoinType(), join.JoinKeys(), ctx); -} + return GetDqJoinResultType<IsMapJoin>(join.Pos(), *leftStructType, leftTableLabels, *rightStructType, + rightTableLabels, join.JoinType(), join.JoinKeys(), ctx); + } } // unnamed @@ -689,12 +722,31 @@ TStatus AnnotateDqCnStreamLookup(const TExprNode::TPtr& input, TExprContext& ctx if (!EnsureStructType(input->Pos(), rightRowType, ctx)) { return TStatus::Error; } + + THashSet<TStringBuf> leftLabels; + if (cnStreamLookup.LeftLabel().Maybe<TCoAtom>()) { + leftLabels.emplace(cnStreamLookup.LeftLabel().Cast<TCoAtom>().Value()); + } else { + for (auto label : cnStreamLookup.LeftLabel().Cast<TCoAtomList>()) { + leftLabels.emplace(label.Value()); + } + } + + THashSet<TStringBuf> rightLabels; + if (cnStreamLookup.RightLabel().Maybe<TCoAtom>()) { + rightLabels.emplace(cnStreamLookup.RightLabel().Cast<TCoAtom>().Value()); + } else { + for (auto label : cnStreamLookup.RightLabel().Cast<TCoAtomList>()) { + rightLabels.emplace(label.Value()); + } + } + const auto outputRowType = GetDqJoinResultType<true>( input->Pos(), *leftRowType.Cast<TStructExprType>(), - cnStreamLookup.LeftLabel().Cast<TCoAtom>().StringValue(), + leftLabels, *rightRowType.Cast<TStructExprType>(), - cnStreamLookup.RightLabel().StringValue(), + rightLabels, cnStreamLookup.JoinType().StringValue(), cnStreamLookup.JoinKeys(), ctx diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_13.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_13.plan index 648ee6407d8..60603637717 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_13.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join1.test_/query_13.plan @@ -5,7 +5,7 @@ "Plans": [ { "Node Type": "ResultSet_1", - "PlanNodeId": 14, + "PlanNodeId": 12, "PlanNodeType": "ResultSet", "Plans": [ { @@ -14,134 +14,101 @@ { "Inputs": [ { - "ExternalPlanNodeId": 12 + "ExternalPlanNodeId": 10 } ], "Limit": "1001", "Name": "Limit" } ], - "PlanNodeId": 13, + "PlanNodeId": 11, "Plans": [ { "Node Type": "UnionAll", - "PlanNodeId": 12, + "PlanNodeId": 10, "PlanNodeType": "Connection", "Plans": [ { "CTE Name": "precompute_0_0", - "Node Type": "Limit-Filter-LeftJoin (MapJoin)-ConstantExpr", + "Node Type": "Limit-InnerJoin (MapJoin)-ConstantExpr-Filter", "Operators": [ { "Inputs": [ { "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 } ], "Limit": "1001", "Name": "Limit" }, { - "Inputs": [ - { - "InternalOperatorId": 2 - } - ], - "Name": "Filter", - "Predicate": "item.J2_TBL.k == 1" - }, - { - "Condition": "J1_TBL.i = J2_TBL.i", + "Condition": "J2_TBL.i = J1_TBL.i", "Inputs": [ { "InternalOperatorId": 3 }, { - "ExternalPlanNodeId": 10 + "InternalOperatorId": 2 } ], - "Name": "LeftJoin (MapJoin)" + "Name": "InnerJoin (MapJoin)" }, { "Inputs": [], "Name": "ToFlow", "ToFlow": "precompute_0_0" + }, + { + "Inputs": [ + { + "ExternalPlanNodeId": 8 + } + ], + "Name": "Filter", + "Predicate": "Exist(item.i) AND item.k == 1" } ], - "PlanNodeId": 11, + "PlanNodeId": 9, "Plans": [ { - "Node Type": "Broadcast", - "PlanNodeId": 10, + "Columns": [ + "i", + "k" + ], + "E-Cost": "No estimate", + "E-Rows": "No estimate", + "E-Size": "No estimate", + "LookupKeyColumns": [ + "i" + ], + "Node Type": "TableLookup", + "Path": "/Root/postgres_jointest/join1.test_plan/J2_TBL", + "PlanNodeId": 8, "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Filter", + "CTE Name": "precompute_0_0", + "Node Type": "ConstantExpr-Aggregate", "Operators": [ { "Inputs": [ { - "ExternalPlanNodeId": 8 + "InternalOperatorId": 1 } ], - "Name": "Filter", - "Predicate": "Exist(item.i)" - } - ], - "PlanNodeId": 9, - "Plans": [ + "Iterator": "PartitionByKey", + "Name": "Iterator" + }, { - "Columns": [ - "i", - "k" - ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", - "LookupKeyColumns": [ - "i" - ], - "Node Type": "TableLookup", - "Path": "/Root/postgres_jointest/join1.test_plan/J2_TBL", - "PlanNodeId": 8, - "PlanNodeType": "Connection", - "Plans": [ - { - "CTE Name": "precompute_0_0", - "Node Type": "ConstantExpr-Aggregate", - "Operators": [ - { - "Inputs": [ - { - "InternalOperatorId": 1 - } - ], - "Iterator": "PartitionByKey", - "Name": "Iterator" - }, - { - "Input": "precompute_0_0", - "Inputs": [], - "Name": "PartitionByKey" - } - ], - "PlanNodeId": 7 - } - ], - "Table": "postgres_jointest/join1.test_plan/J2_TBL" + "Input": "precompute_0_0", + "Inputs": [], + "Name": "PartitionByKey" } - ] + ], + "PlanNodeId": 7 } - ] + ], + "Table": "postgres_jointest/join1.test_plan/J2_TBL" } ] } diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_11.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_11.plan index 76d9b112d4d..a4b4fd56b91 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_11.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_11.plan @@ -5,7 +5,7 @@ "Plans": [ { "Node Type": "ResultSet_1", - "PlanNodeId": 19, + "PlanNodeId": 17, "PlanNodeType": "ResultSet", "Plans": [ { @@ -14,74 +14,38 @@ { "Inputs": [ { - "ExternalPlanNodeId": 17 + "ExternalPlanNodeId": 15 } ], "Limit": "1001", "Name": "Limit" } ], - "PlanNodeId": 18, + "PlanNodeId": 16, "Plans": [ { "Node Type": "Merge", - "PlanNodeId": 17, + "PlanNodeId": 15, "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "TopSort-Filter-LeftJoin (MapJoin)", + "Node Type": "TopSort-LeftJoin (MapJoin)", "Operators": [ { "Inputs": [ { "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 } ], "Limit": "1001", "Name": "TopSort", - "TopSortBy": "row.x1" - }, - { - "Inputs": [ - { - "InternalOperatorId": 2 - } - ], - "Name": "Filter", - "Predicate": "Exist(item.y.y2)" + "TopSortBy": "argument.x_1.x1" }, { "Condition": "x_1.x1 = xx.x1", "Inputs": [ { - "ExternalPlanNodeId": 15 + "ExternalPlanNodeId": 13 }, { "ExternalPlanNodeId": 9 @@ -90,100 +54,85 @@ "Name": "LeftJoin (MapJoin)" } ], - "PlanNodeId": 16, + "PlanNodeId": 14, "Plans": [ { "Node Type": "Map", - "PlanNodeId": 15, + "PlanNodeId": 13, "PlanNodeType": "Connection", "Plans": [ { "CTE Name": "precompute_0_0", - "Node Type": "LeftJoin (MapJoin)-ConstantExpr", + "Node Type": "InnerJoin (MapJoin)-ConstantExpr-Filter", "Operators": [ { - "Condition": "x_1.x1 = y.y1", + "Condition": "y.y1 = x_1.x1", "Inputs": [ { - "InternalOperatorId": 1 + "InternalOperatorId": 2 }, { - "ExternalPlanNodeId": 13 + "InternalOperatorId": 1 } ], - "Name": "LeftJoin (MapJoin)" + "Name": "InnerJoin (MapJoin)" }, { "Inputs": [], "Name": "ToFlow", "ToFlow": "precompute_0_0" + }, + { + "Inputs": [ + { + "ExternalPlanNodeId": 11 + } + ], + "Name": "Filter", + "Predicate": "Exist(item.y1) AND Exist(item.y2)" } ], - "PlanNodeId": 14, + "PlanNodeId": 12, "Plans": [ { - "Node Type": "Broadcast", - "PlanNodeId": 13, + "Columns": [ + "y1", + "y2" + ], + "E-Cost": "No estimate", + "E-Rows": "No estimate", + "E-Size": "No estimate", + "LookupKeyColumns": [ + "y1" + ], + "Node Type": "TableLookup", + "Path": "/Root/postgres_jointest/join2.test_plan/y", + "PlanNodeId": 11, "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Filter", + "CTE Name": "precompute_0_0", + "Node Type": "ConstantExpr-Aggregate", "Operators": [ { "Inputs": [ { - "ExternalPlanNodeId": 11 + "InternalOperatorId": 1 } ], - "Name": "Filter", - "Predicate": "Exist(item.y1)" - } - ], - "PlanNodeId": 12, - "Plans": [ + "Iterator": "PartitionByKey", + "Name": "Iterator" + }, { - "Columns": [ - "y1", - "y2" - ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", - "LookupKeyColumns": [ - "y1" - ], - "Node Type": "TableLookup", - "Path": "/Root/postgres_jointest/join2.test_plan/y", - "PlanNodeId": 11, - "PlanNodeType": "Connection", - "Plans": [ - { - "CTE Name": "precompute_0_0", - "Node Type": "ConstantExpr-Aggregate", - "Operators": [ - { - "Inputs": [ - { - "InternalOperatorId": 1 - } - ], - "Iterator": "PartitionByKey", - "Name": "Iterator" - }, - { - "Input": "precompute_0_0", - "Inputs": [], - "Name": "PartitionByKey" - } - ], - "PlanNodeId": 10 - } - ], - "Table": "postgres_jointest/join2.test_plan/y" + "Input": "precompute_0_0", + "Inputs": [], + "Name": "PartitionByKey" } - ] + ], + "PlanNodeId": 10 } - ] + ], + "Table": "postgres_jointest/join2.test_plan/y" } ] } @@ -231,7 +180,7 @@ } ], "SortColumns": [ - "x1 (Asc)" + "x_1.x1 (Asc)" ] } ] diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_12.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_12.plan index 78f73d0236a..d022a4ce52b 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_12.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_12.plan @@ -4,8 +4,8 @@ "PlanNodeType": "Query", "Plans": [ { - "Node Type": "ResultSet_1", - "PlanNodeId": 19, + "Node Type": "ResultSet_2", + "PlanNodeId": 21, "PlanNodeType": "ResultSet", "Plans": [ { @@ -14,214 +14,212 @@ { "Inputs": [ { - "ExternalPlanNodeId": 17 + "ExternalPlanNodeId": 19 } ], "Limit": "1001", "Name": "Limit" } ], - "PlanNodeId": 18, + "PlanNodeId": 20, "Plans": [ { "Node Type": "Merge", - "PlanNodeId": 17, + "PlanNodeId": 19, "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "TopSort-LeftJoin (MapJoin)", + "CTE Name": "precompute_1_0", + "Node Type": "TopSort-InnerJoin (MapJoin)-ConstantExpr-Filter", "Operators": [ { "Inputs": [ { "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 } ], "Limit": "1001", "Name": "TopSort", - "TopSortBy": "row.x1" + "TopSortBy": "argument.x_1.x1" }, { - "Condition": "x_1.x1 = xx.x1", + "Condition": "xx.x1 = x_1.x1", "Inputs": [ { - "ExternalPlanNodeId": 15 + "InternalOperatorId": 3 }, { - "ExternalPlanNodeId": 9 + "InternalOperatorId": 2 } ], - "Name": "LeftJoin (MapJoin)" + "Name": "InnerJoin (MapJoin)" + }, + { + "Inputs": [], + "Name": "ToFlow", + "ToFlow": "precompute_1_0" + }, + { + "Inputs": [ + { + "ExternalPlanNodeId": 17 + } + ], + "Name": "Filter", + "Predicate": "Exist(item.x1) AND Exist(item.x2)" } ], - "PlanNodeId": 16, + "PlanNodeId": 18, "Plans": [ { - "Node Type": "Map", - "PlanNodeId": 15, + "Columns": [ + "x1", + "x2" + ], + "E-Cost": "No estimate", + "E-Rows": "No estimate", + "E-Size": "No estimate", + "LookupKeyColumns": [ + "x1" + ], + "Node Type": "TableLookup", + "Path": "/Root/postgres_jointest/join2.test_plan/x", + "PlanNodeId": 17, "PlanNodeType": "Connection", "Plans": [ { - "CTE Name": "precompute_0_0", - "Node Type": "LeftJoin (MapJoin)-ConstantExpr", + "CTE Name": "precompute_1_0", + "Node Type": "ConstantExpr-Aggregate", "Operators": [ { - "Condition": "x_1.x1 = y.y1", "Inputs": [ { "InternalOperatorId": 1 - }, - { - "ExternalPlanNodeId": 13 } ], - "Name": "LeftJoin (MapJoin)" + "Iterator": "PartitionByKey", + "Name": "Iterator" }, { + "Input": "precompute_1_0", "Inputs": [], - "Name": "ToFlow", - "ToFlow": "precompute_0_0" + "Name": "PartitionByKey" } ], - "PlanNodeId": 14, + "PlanNodeId": 16 + } + ], + "Table": "postgres_jointest/join2.test_plan/x" + } + ] + } + ], + "SortColumns": [ + "x_1.x1 (Asc)" + ] + } + ] + } + ] + }, + { + "Node Type": "Precompute_1", + "Parent Relationship": "InitPlan", + "PlanNodeId": 14, + "PlanNodeType": "Materialize", + "Plans": [ + { + "Node Type": "Collect", + "PlanNodeId": 13, + "Plans": [ + { + "Node Type": "UnionAll", + "PlanNodeId": 12, + "PlanNodeType": "Connection", + "Plans": [ + { + "CTE Name": "precompute_0_0", + "Node Type": "LeftJoin (MapJoin)-ConstantExpr", + "Operators": [ + { + "Condition": "x_1.x1 = y.y1", + "Inputs": [ + { + "InternalOperatorId": 1 + }, + { + "ExternalPlanNodeId": 10 + } + ], + "Name": "LeftJoin (MapJoin)" + }, + { + "Inputs": [], + "Name": "ToFlow", + "ToFlow": "precompute_0_0" + } + ], + "PlanNodeId": 11, + "Plans": [ + { + "Node Type": "Broadcast", + "PlanNodeId": 10, + "PlanNodeType": "Connection", + "Plans": [ + { + "Node Type": "Filter", + "Operators": [ + { + "Inputs": [ + { + "ExternalPlanNodeId": 8 + } + ], + "Name": "Filter", + "Predicate": "Exist(item.y1)" + } + ], + "PlanNodeId": 9, "Plans": [ { - "Node Type": "Broadcast", - "PlanNodeId": 13, + "Columns": [ + "y1", + "y2" + ], + "E-Cost": "No estimate", + "E-Rows": "No estimate", + "E-Size": "No estimate", + "LookupKeyColumns": [ + "y1" + ], + "Node Type": "TableLookup", + "Path": "/Root/postgres_jointest/join2.test_plan/y", + "PlanNodeId": 8, "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Filter", + "CTE Name": "precompute_0_0", + "Node Type": "ConstantExpr-Aggregate", "Operators": [ { "Inputs": [ { - "ExternalPlanNodeId": 11 + "InternalOperatorId": 1 } ], - "Name": "Filter", - "Predicate": "Exist(item.y1)" - } - ], - "PlanNodeId": 12, - "Plans": [ + "Iterator": "PartitionByKey", + "Name": "Iterator" + }, { - "Columns": [ - "y1", - "y2" - ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", - "LookupKeyColumns": [ - "y1" - ], - "Node Type": "TableLookup", - "Path": "/Root/postgres_jointest/join2.test_plan/y", - "PlanNodeId": 11, - "PlanNodeType": "Connection", - "Plans": [ - { - "CTE Name": "precompute_0_0", - "Node Type": "ConstantExpr-Aggregate", - "Operators": [ - { - "Inputs": [ - { - "InternalOperatorId": 1 - } - ], - "Iterator": "PartitionByKey", - "Name": "Iterator" - }, - { - "Input": "precompute_0_0", - "Inputs": [], - "Name": "PartitionByKey" - } - ], - "PlanNodeId": 10 - } - ], - "Table": "postgres_jointest/join2.test_plan/y" + "Input": "precompute_0_0", + "Inputs": [], + "Name": "PartitionByKey" } - ] - } - ] - } - ] - } - ] - }, - { - "Node Type": "Broadcast", - "PlanNodeId": 9, - "PlanNodeType": "Connection", - "Plans": [ - { - "Node Type": "Stage", - "PlanNodeId": 8, - "Plans": [ - { - "Node Type": "TableFullScan", - "Operators": [ - { - "Inputs": [], - "Name": "TableFullScan", - "Path": "/Root/postgres_jointest/join2.test_plan/x", - "ReadColumns": [ - "x1", - "x2" - ], - "ReadRanges": [ - "x1 (-\u221e, +\u221e)", - "x2 (-\u221e, +\u221e)" ], - "ReadRangesPointPrefixLen": "0", - "Scan": "Parallel", - "Table": "postgres_jointest/join2.test_plan/x" + "PlanNodeId": 7 } ], - "PlanNodeId": 7, - "Tables": [ - "postgres_jointest/join2.test_plan/x" - ] + "Table": "postgres_jointest/join2.test_plan/y" } ] } @@ -229,14 +227,12 @@ } ] } - ], - "SortColumns": [ - "x1 (Asc)" ] } ] } - ] + ], + "Subplan Name": "CTE precompute_1_0" }, { "Node Type": "Precompute_0", @@ -320,11 +316,10 @@ "x1", "x2" ], - "scan_by": [ - "x1 (-\u221e, +\u221e)", - "x2 (-\u221e, +\u221e)" + "lookup_by": [ + "x1" ], - "type": "FullScan" + "type": "Lookup" } ] }, diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_8.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_8.plan index bc8e818177e..0dce147613e 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_8.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join2.test_/query_8.plan @@ -5,7 +5,7 @@ "Plans": [ { "Node Type": "ResultSet_1", - "PlanNodeId": 14, + "PlanNodeId": 12, "PlanNodeType": "ResultSet", "Plans": [ { @@ -14,144 +14,111 @@ { "Inputs": [ { - "ExternalPlanNodeId": 12 + "ExternalPlanNodeId": 10 } ], "Limit": "1001", "Name": "Limit" } ], - "PlanNodeId": 13, + "PlanNodeId": 11, "Plans": [ { "Node Type": "Merge", - "PlanNodeId": 12, + "PlanNodeId": 10, "PlanNodeType": "Connection", "Plans": [ { "CTE Name": "precompute_0_0", - "Node Type": "TopSort-Filter-LeftJoin (MapJoin)-ConstantExpr", + "Node Type": "TopSort-InnerJoin (MapJoin)-ConstantExpr-Filter", "Operators": [ { "Inputs": [ { "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 } ], "Limit": "1001", "Name": "TopSort", - "TopSortBy": "[row.x1,row.x2,row.y1,row.y2]" + "TopSortBy": "[Arg.x.x1,Arg.x.x2,Arg.y.y1,Arg.y.y2]" }, { - "Inputs": [ - { - "InternalOperatorId": 2 - } - ], - "Name": "Filter", - "Predicate": "Exist(item.y.y2)" - }, - { - "Condition": "x.x1 = y.y1", + "Condition": "y.y1 = x.x1", "Inputs": [ { "InternalOperatorId": 3 }, { - "ExternalPlanNodeId": 10 + "InternalOperatorId": 2 } ], - "Name": "LeftJoin (MapJoin)" + "Name": "InnerJoin (MapJoin)" }, { "Inputs": [], "Name": "ToFlow", "ToFlow": "precompute_0_0" + }, + { + "Inputs": [ + { + "ExternalPlanNodeId": 8 + } + ], + "Name": "Filter", + "Predicate": "Exist(item.y1) AND Exist(item.y2)" } ], - "PlanNodeId": 11, + "PlanNodeId": 9, "Plans": [ { - "Node Type": "Broadcast", - "PlanNodeId": 10, + "Columns": [ + "y1", + "y2" + ], + "E-Cost": "No estimate", + "E-Rows": "No estimate", + "E-Size": "No estimate", + "LookupKeyColumns": [ + "y1" + ], + "Node Type": "TableLookup", + "Path": "/Root/postgres_jointest/join2.test_plan/y", + "PlanNodeId": 8, "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Filter", + "CTE Name": "precompute_0_0", + "Node Type": "ConstantExpr-Aggregate", "Operators": [ { "Inputs": [ { - "ExternalPlanNodeId": 8 + "InternalOperatorId": 1 } ], - "Name": "Filter", - "Predicate": "Exist(item.y1)" - } - ], - "PlanNodeId": 9, - "Plans": [ + "Iterator": "PartitionByKey", + "Name": "Iterator" + }, { - "Columns": [ - "y1", - "y2" - ], - "E-Cost": "No estimate", - "E-Rows": "No estimate", - "E-Size": "No estimate", - "LookupKeyColumns": [ - "y1" - ], - "Node Type": "TableLookup", - "Path": "/Root/postgres_jointest/join2.test_plan/y", - "PlanNodeId": 8, - "PlanNodeType": "Connection", - "Plans": [ - { - "CTE Name": "precompute_0_0", - "Node Type": "ConstantExpr-Aggregate", - "Operators": [ - { - "Inputs": [ - { - "InternalOperatorId": 1 - } - ], - "Iterator": "PartitionByKey", - "Name": "Iterator" - }, - { - "Input": "precompute_0_0", - "Inputs": [], - "Name": "PartitionByKey" - } - ], - "PlanNodeId": 7 - } - ], - "Table": "postgres_jointest/join2.test_plan/y" + "Input": "precompute_0_0", + "Inputs": [], + "Name": "PartitionByKey" } - ] + ], + "PlanNodeId": 7 } - ] + ], + "Table": "postgres_jointest/join2.test_plan/y" } ] } ], "SortColumns": [ - "x1 (Asc)", - "x2 (Asc)", - "y1 (Asc)", - "y2 (Asc)" + "x.x1 (Asc)", + "x.x2 (Asc)", + "y.y1 (Asc)", + "y.y2 (Asc)" ] } ] diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_1.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_1.plan index 74db34123d3..482fedb9389 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_1.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join3.test_/query_1.plan @@ -4,8 +4,8 @@ "PlanNodeType": "Query", "Plans": [ { - "Node Type": "ResultSet", - "PlanNodeId": 10, + "Node Type": "ResultSet_1", + "PlanNodeId": 26, "PlanNodeType": "ResultSet", "Plans": [ { @@ -14,142 +14,374 @@ { "Inputs": [ { - "ExternalPlanNodeId": 8 + "ExternalPlanNodeId": 24 } ], "Limit": "1001", "Name": "Limit" } ], - "PlanNodeId": 9, + "PlanNodeId": 25, "Plans": [ { - "Node Type": "Merge", - "PlanNodeId": 8, + "Node Type": "UnionAll", + "PlanNodeId": 24, "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "TopSort-Filter-LeftJoin (MapJoin)", + "Node Type": "Limit", "Operators": [ { "Inputs": [ { - "InternalOperatorId": 1 - }, - { - "InternalOperatorId": 1 + "ExternalPlanNodeId": 22 } ], "Limit": "1001", - "Name": "TopSort", - "TopSortBy": "[row.q2,row.q1]" - }, - { - "Inputs": [ - { - "InternalOperatorId": 2 - } - ], - "Name": "Filter", - "Predicate": "item.b.q1 > 0" - }, - { - "Condition": "a.q2 = b._equijoin_column_0", - "Inputs": [ - { - "ExternalPlanNodeId": 6 - }, - { - "ExternalPlanNodeId": 3 - } - ], - "Name": "LeftJoin (MapJoin)" + "Name": "Limit" } ], - "PlanNodeId": 7, + "PlanNodeId": 23, "Plans": [ { - "Node Type": "Map", - "PlanNodeId": 6, + "Node Type": "Merge", + "PlanNodeId": 22, "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Stage", - "PlanNodeId": 5, - "Plans": [ + "Node Type": "Sort-Union", + "Operators": [ { - "Node Type": "TableFullScan", - "Operators": [ + "Inputs": [ { - "Inputs": [], - "Name": "TableFullScan", - "Path": "/Root/postgres_jointest/join3.test_plan/int8_tbl", - "ReadColumns": [ - "q1", - "q2" - ], - "ReadRanges": [ - "q1 (-\u221e, +\u221e)", - "q2 (-\u221e, +\u221e)" - ], - "ReadRangesPointPrefixLen": "0", - "Scan": "Parallel", - "Table": "postgres_jointest/join3.test_plan/int8_tbl" + "InternalOperatorId": 1 } ], - "PlanNodeId": 4, - "Tables": [ - "postgres_jointest/join3.test_plan/int8_tbl" - ] + "Name": "Sort", + "SortBy": "[row.q2,row.q1]" + }, + { + "Inputs": [ + { + "ExternalPlanNodeId": 20 + }, + { + "ExternalPlanNodeId": 10 + } + ], + "Name": "Union" } - ] - } - ] - }, - { - "Node Type": "Broadcast", - "PlanNodeId": 3, - "PlanNodeType": "Connection", - "Plans": [ - { - "Node Type": "Stage", - "PlanNodeId": 2, + ], + "PlanNodeId": 21, "Plans": [ { - "Node Type": "TableFullScan", - "Operators": [ + "Node Type": "UnionAll", + "PlanNodeId": 20, + "PlanNodeType": "Connection", + "Plans": [ { - "Inputs": [], - "Name": "TableFullScan", - "Path": "/Root/postgres_jointest/join3.test_plan/int8_tbl", - "ReadColumns": [ - "q1", - "q2" + "Node Type": "Top", + "Operators": [ + { + "Inputs": [ + { + "ExternalPlanNodeId": 18 + } + ], + "Limit": "1001", + "Name": "Top", + "TopBy": "[row.q2,row.q1]" + } ], - "ReadRanges": [ - "q1 (-\u221e, +\u221e)", - "q2 (-\u221e, +\u221e)" + "PlanNodeId": 19, + "Plans": [ + { + "Node Type": "UnionAll", + "PlanNodeId": 18, + "PlanNodeType": "Connection", + "Plans": [ + { + "Node Type": "Top-InnerJoin (MapJoin)-Filter", + "Operators": [ + { + "Inputs": [ + { + "InternalOperatorId": 1 + }, + { + "InternalOperatorId": 1 + } + ], + "Limit": "1001", + "Name": "Top", + "TopBy": "[row.q2,row.q1]" + }, + { + "Condition": "a.q2 = b._equijoin_column_0", + "Inputs": [ + { + "InternalOperatorId": 2 + }, + { + "ExternalPlanNodeId": 13 + } + ], + "Name": "InnerJoin (MapJoin)" + }, + { + "Inputs": [ + { + "ExternalPlanNodeId": 16 + } + ], + "Name": "Filter", + "Predicate": "Exist(item.q2)" + } + ], + "PlanNodeId": 17, + "Plans": [ + { + "Node Type": "Map", + "PlanNodeId": 16, + "PlanNodeType": "Connection", + "Plans": [ + { + "Node Type": "Stage", + "PlanNodeId": 15, + "Plans": [ + { + "Node Type": "TableFullScan", + "Operators": [ + { + "Inputs": [], + "Name": "TableFullScan", + "Path": "/Root/postgres_jointest/join3.test_plan/int8_tbl", + "ReadColumns": [ + "q1", + "q2" + ], + "ReadRanges": [ + "q1 (-\u221e, +\u221e)", + "q2 (-\u221e, +\u221e)" + ], + "ReadRangesPointPrefixLen": "0", + "Scan": "Parallel", + "Table": "postgres_jointest/join3.test_plan/int8_tbl" + } + ], + "PlanNodeId": 14, + "Tables": [ + "postgres_jointest/join3.test_plan/int8_tbl" + ] + } + ] + } + ] + }, + { + "Node Type": "Broadcast", + "PlanNodeId": 13, + "PlanNodeType": "Connection", + "Plans": [ + { + "Node Type": "Stage", + "PlanNodeId": 12, + "Plans": [ + { + "Node Type": "TableFullScan", + "Operators": [ + { + "Inputs": [], + "Name": "TableFullScan", + "Path": "/Root/postgres_jointest/join3.test_plan/int8_tbl", + "ReadColumns": [ + "q1", + "q2" + ], + "ReadRanges": [ + "q1 (-\u221e, +\u221e)", + "q2 (-\u221e, +\u221e)" + ], + "ReadRangesPointPrefixLen": "0", + "Scan": "Parallel", + "Table": "postgres_jointest/join3.test_plan/int8_tbl" + } + ], + "PlanNodeId": 11, + "Tables": [ + "postgres_jointest/join3.test_plan/int8_tbl" + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + }, + { + "Node Type": "UnionAll", + "PlanNodeId": 10, + "PlanNodeType": "Connection", + "Plans": [ + { + "Node Type": "Top", + "Operators": [ + { + "Inputs": [ + { + "ExternalPlanNodeId": 8 + } + ], + "Limit": "1001", + "Name": "Top", + "TopBy": "[row.q2,row.q1]" + } ], - "ReadRangesPointPrefixLen": "0", - "Scan": "Parallel", - "Table": "postgres_jointest/join3.test_plan/int8_tbl" + "PlanNodeId": 9, + "Plans": [ + { + "Node Type": "UnionAll", + "PlanNodeId": 8, + "PlanNodeType": "Connection", + "Plans": [ + { + "Node Type": "Top-LeftOnlyJoin (MapJoin)", + "Operators": [ + { + "Inputs": [ + { + "InternalOperatorId": 1 + } + ], + "Limit": "1001", + "Name": "Top", + "TopBy": "[row.q2,row.q1]" + }, + { + "Condition": "a.q2 = b._equijoin_column_0", + "Inputs": [ + { + "ExternalPlanNodeId": 6 + }, + { + "ExternalPlanNodeId": 3 + } + ], + "Name": "LeftOnlyJoin (MapJoin)" + } + ], + "PlanNodeId": 7, + "Plans": [ + { + "Node Type": "Map", + "PlanNodeId": 6, + "PlanNodeType": "Connection", + "Plans": [ + { + "Node Type": "Stage", + "PlanNodeId": 5, + "Plans": [ + { + "Node Type": "TableRangeScan", + "Operators": [ + { + "Inputs": [], + "Name": "TableRangeScan", + "Path": "/Root/postgres_jointest/join3.test_plan/int8_tbl", + "ReadColumns": [ + "q2" + ], + "ReadRangesExpectedSize": "1", + "ReadRangesKeys": [ + "q1" + ], + "ReadRangesPointPrefixLen": "0", + "Scan": "Parallel", + "Table": "postgres_jointest/join3.test_plan/int8_tbl" + } + ], + "PlanNodeId": 4, + "Tables": [ + "postgres_jointest/join3.test_plan/int8_tbl" + ] + } + ] + } + ] + }, + { + "Node Type": "Broadcast", + "PlanNodeId": 3, + "PlanNodeType": "Connection", + "Plans": [ + { + "Node Type": "Filter", + "Operators": [ + { + "Inputs": [ + { + "ExternalPlanNodeId": 1 + } + ], + "Name": "Filter", + "Predicate": "1 > 0" + } + ], + "PlanNodeId": 2, + "Plans": [ + { + "Node Type": "TableFullScan", + "Operators": [ + { + "Inputs": [], + "Name": "TableFullScan", + "Path": "/Root/postgres_jointest/join3.test_plan/int8_tbl", + "ReadColumns": [ + "q1", + "q2" + ], + "ReadRanges": [ + "q1 (-\u221e, +\u221e)", + "q2 (-\u221e, +\u221e)" + ], + "ReadRangesPointPrefixLen": "0", + "Scan": "Parallel", + "Table": "postgres_jointest/join3.test_plan/int8_tbl" + } + ], + "PlanNodeId": 1, + "Tables": [ + "postgres_jointest/join3.test_plan/int8_tbl" + ] + } + ] + } + ] + } + ] + } + ] + } + ] } - ], - "PlanNodeId": 1, - "Tables": [ - "postgres_jointest/join3.test_plan/int8_tbl" ] } ] } + ], + "SortColumns": [ + "q2 (Asc)", + "q1 (Asc)" ] } ] } - ], - "SortColumns": [ - "q2 (Asc)", - "q1 (Asc)" ] } ] @@ -190,6 +422,23 @@ "q2 (-\u221e, +\u221e)" ], "type": "FullScan" + }, + { + "columns": [ + "q1", + "q2" + ], + "scan_by": [ + "q1 (-\u221e, +\u221e)", + "q2 (-\u221e, +\u221e)" + ], + "type": "FullScan" + }, + { + "columns": [ + "q2" + ], + "type": "Scan" } ] } diff --git a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_1.plan b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_1.plan index f60f512526a..0e248ab4d09 100644 --- a/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_1.plan +++ b/ydb/tests/functional/suite_tests/canondata/test_postgres.TestPGSQL.test_sql_suite_plan-jointest_join4.test_/query_1.plan @@ -4,8 +4,8 @@ "PlanNodeType": "Query", "Plans": [ { - "Node Type": "ResultSet", - "PlanNodeId": 11, + "Node Type": "ResultSet_1", + "PlanNodeId": 15, "PlanNodeType": "ResultSet", "Plans": [ { @@ -14,22 +14,22 @@ { "Inputs": [ { - "ExternalPlanNodeId": 9 + "ExternalPlanNodeId": 13 } ], "Limit": "1001", "Name": "Limit" } ], - "PlanNodeId": 10, + "PlanNodeId": 14, "Plans": [ { "Node Type": "UnionAll", - "PlanNodeId": 9, + "PlanNodeId": 13, "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Limit-Filter-LeftJoin (MapJoin)", + "Node Type": "Limit-InnerJoin (MapJoin)-Filter", "Operators": [ { "Inputs": [ @@ -41,28 +41,28 @@ "Name": "Limit" }, { + "Condition": "nt3.nt2_id = nt2.id", "Inputs": [ { "InternalOperatorId": 2 + }, + { + "ExternalPlanNodeId": 10 } ], - "Name": "Filter", - "Predicate": "If" + "Name": "InnerJoin (MapJoin)" }, { - "Condition": "nt3.nt2_id = ss2.nt2.id", "Inputs": [ { - "ExternalPlanNodeId": 7 - }, - { - "ExternalPlanNodeId": 6 + "ExternalPlanNodeId": 11 } ], - "Name": "LeftJoin (MapJoin)" + "Name": "Filter", + "Predicate": "Exist(item.nt2_id)" } ], - "PlanNodeId": 8, + "PlanNodeId": 12, "Plans": [ { "Node Type": "TablePointLookup", @@ -82,94 +82,87 @@ "Table": "postgres_jointest/join4.test_plan/nt3" } ], - "PlanNodeId": 7, + "PlanNodeId": 11, "Tables": [ "postgres_jointest/join4.test_plan/nt3" ] }, { "Node Type": "Broadcast", - "PlanNodeId": 6, + "PlanNodeId": 10, "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "LeftJoin (MapJoin)", + "CTE Name": "precompute_0_0", + "Node Type": "InnerJoin (MapJoin)-ConstantExpr-Filter", "Operators": [ { - "Condition": "nt2.nt1_id = ss1.id", + "Condition": "ss1.id = nt2.nt1_id", "Inputs": [ { - "ExternalPlanNodeId": 4 + "InternalOperatorId": 2 }, { - "ExternalPlanNodeId": 3 + "InternalOperatorId": 1 } ], - "Name": "LeftJoin (MapJoin)" - } - ], - "PlanNodeId": 5, - "Plans": [ + "Name": "InnerJoin (MapJoin)" + }, + { + "Inputs": [], + "Name": "ToFlow", + "ToFlow": "precompute_0_0" + }, { - "Node Type": "TableFullScan", - "Operators": [ + "Inputs": [ { - "Inputs": [], - "Name": "TableFullScan", - "Path": "/Root/postgres_jointest/join4.test_plan/nt2", - "ReadColumns": [ - "b1", - "id", - "nt1_id" - ], - "ReadRanges": [ - "id (-\u221e, +\u221e)" - ], - "ReadRangesPointPrefixLen": "0", - "Scan": "Parallel", - "Table": "postgres_jointest/join4.test_plan/nt2" + "ExternalPlanNodeId": 8 } ], - "PlanNodeId": 4, - "Tables": [ - "postgres_jointest/join4.test_plan/nt2" - ] - }, + "Name": "Filter", + "Predicate": "Exist(item.id)" + } + ], + "PlanNodeId": 9, + "Plans": [ { - "Node Type": "Broadcast", - "PlanNodeId": 3, + "Columns": [ + "id" + ], + "E-Cost": "No estimate", + "E-Rows": "No estimate", + "E-Size": "No estimate", + "LookupKeyColumns": [ + "id" + ], + "Node Type": "TableLookup", + "Path": "/Root/postgres_jointest/join4.test_plan/nt1", + "PlanNodeId": 8, "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Stage", - "PlanNodeId": 2, - "Plans": [ + "CTE Name": "precompute_0_0", + "Node Type": "ConstantExpr-Aggregate", + "Operators": [ { - "Node Type": "TableFullScan", - "Operators": [ + "Inputs": [ { - "Inputs": [], - "Name": "TableFullScan", - "Path": "/Root/postgres_jointest/join4.test_plan/nt1", - "ReadColumns": [ - "id" - ], - "ReadRanges": [ - "id (-\u221e, +\u221e)" - ], - "ReadRangesPointPrefixLen": "0", - "Scan": "Parallel", - "Table": "postgres_jointest/join4.test_plan/nt1" + "InternalOperatorId": 1 } ], - "PlanNodeId": 1, - "Tables": [ - "postgres_jointest/join4.test_plan/nt1" - ] + "Iterator": "PartitionByKey", + "Name": "Iterator" + }, + { + "Input": "precompute_0_0", + "Inputs": [], + "Name": "PartitionByKey" } - ] + ], + "PlanNodeId": 7 } - ] + ], + "Table": "postgres_jointest/join4.test_plan/nt1" } ] } @@ -182,6 +175,70 @@ ] } ] + }, + { + "Node Type": "Precompute_0", + "Parent Relationship": "InitPlan", + "PlanNodeId": 5, + "PlanNodeType": "Materialize", + "Plans": [ + { + "Node Type": "Collect", + "PlanNodeId": 4, + "Plans": [ + { + "Node Type": "UnionAll", + "PlanNodeId": 3, + "PlanNodeType": "Connection", + "Plans": [ + { + "Node Type": "Filter", + "Operators": [ + { + "Inputs": [ + { + "ExternalPlanNodeId": 1 + } + ], + "Name": "Filter", + "Predicate": "item.b1" + } + ], + "PlanNodeId": 2, + "Plans": [ + { + "Node Type": "TableFullScan", + "Operators": [ + { + "Inputs": [], + "Name": "TableFullScan", + "Path": "/Root/postgres_jointest/join4.test_plan/nt2", + "ReadColumns": [ + "b1", + "id", + "nt1_id" + ], + "ReadRanges": [ + "id (-\u221e, +\u221e)" + ], + "ReadRangesPointPrefixLen": "0", + "Scan": "Parallel", + "Table": "postgres_jointest/join4.test_plan/nt2" + } + ], + "PlanNodeId": 1, + "Tables": [ + "postgres_jointest/join4.test_plan/nt2" + ] + } + ] + } + ] + } + ] + } + ], + "Subplan Name": "CTE precompute_0_0" } ], "Stats": { @@ -200,10 +257,10 @@ "columns": [ "id" ], - "scan_by": [ - "id (-\u221e, +\u221e)" + "lookup_by": [ + "id" ], - "type": "FullScan" + "type": "Lookup" } ] }, |