diff options
author | aozeritsky <aozeritsky@ydb.tech> | 2023-10-02 19:04:14 +0300 |
---|---|---|
committer | aozeritsky <aozeritsky@ydb.tech> | 2023-10-02 20:02:21 +0300 |
commit | 50fcf535f7e8fb0c1660d743c46a4e79500f326d (patch) | |
tree | 0c09bf85f417bff17aa38a86b40184f2ce8a23c3 | |
parent | b0c2958279bf47a7c37cf1068bf2af2c07dd5e30 (diff) | |
download | ydb-50fcf535f7e8fb0c1660d743c46a4e79500f326d.tar.gz |
Remove pragma yt.CostBasedOptimizer
11 files changed, 43 insertions, 42 deletions
diff --git a/ydb/library/yql/core/yql_type_annotation.h b/ydb/library/yql/core/yql_type_annotation.h index de931c1034..1049da8143 100644 --- a/ydb/library/yql/core/yql_type_annotation.h +++ b/ydb/library/yql/core/yql_type_annotation.h @@ -176,6 +176,12 @@ enum class EFallbackPolicy { Always /* "always" */ }; +enum class ECostBasedOptimizerType { + Disable /* "disable" */, + PG /* "pg" */, + Native /* "native" */ +}; + struct TUdfCachedInfo { const TTypeAnnotationNode* FunctionType = nullptr; const TTypeAnnotationNode* RunConfigType = nullptr; @@ -236,7 +242,7 @@ struct TTypeAnnotationContext: public TThrRefBase { bool UseBlocks = false; bool PgEmitAggApply = false; IArrowResolver::TPtr ArrowResolver; - TString CostBasedOptimizerType; + ECostBasedOptimizerType CostBasedOptimizer = ECostBasedOptimizerType::Disable; bool MatchRecognize = false; enum class EMatchRecognizeStreamingMode { Disable, diff --git a/ydb/library/yql/providers/config/yql_config_provider.cpp b/ydb/library/yql/providers/config/yql_config_provider.cpp index 5b531dc2a3..9507a7328b 100644 --- a/ydb/library/yql/providers/config/yql_config_provider.cpp +++ b/ydb/library/yql/providers/config/yql_config_provider.cpp @@ -812,14 +812,11 @@ namespace { ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at most 1 argument, but got " << args.size())); return false; } - auto arg = TString{args[0]}; - if (!(arg == "pg" || arg == "native" || arg == "disable")) { + if (!TryFromString(args[0], Types.CostBasedOptimizer)) { ctx.AddError(TIssue(pos, TStringBuilder() << "Expected `disable|pg|native', but got: " << args[0])); return false; } - - Types.CostBasedOptimizerType = arg; } else if (name == "_EnableMatchRecognize" || name == "DisableMatchRecognize") { if (args.size() != 0) { diff --git a/ydb/library/yql/providers/dq/opt/logical_optimize.cpp b/ydb/library/yql/providers/dq/opt/logical_optimize.cpp index ac1594cffa..90952bd879 100644 --- a/ydb/library/yql/providers/dq/opt/logical_optimize.cpp +++ b/ydb/library/yql/providers/dq/opt/logical_optimize.cpp @@ -116,14 +116,14 @@ protected: } TMaybeNode<TExprBase> OptimizeEquiJoinWithCosts(TExprBase node, TExprContext& ctx) { - if (TypesCtx.CostBasedOptimizerType == "native" || TypesCtx.CostBasedOptimizerType == "pg") { + if (TypesCtx.CostBasedOptimizer != ECostBasedOptimizerType::Disable) { std::function<void(const TString&)> log = [&](auto str) { YQL_CLOG(INFO, ProviderDq) << str; }; std::function<IOptimizer*(IOptimizer::TInput&&)> factory = [&](auto input) { - if (TypesCtx.CostBasedOptimizerType == "native") { + if (TypesCtx.CostBasedOptimizer == ECostBasedOptimizerType::Native) { return MakeNativeOptimizer(input, log); - } else if (TypesCtx.CostBasedOptimizerType == "pg") { + } else if (TypesCtx.CostBasedOptimizer == ECostBasedOptimizerType::PG) { return MakePgOptimizer(input, log); } else { YQL_ENSURE(false, "Unknown CBO type"); diff --git a/ydb/library/yql/providers/yt/common/yql_yt_settings.cpp b/ydb/library/yql/providers/yt/common/yql_yt_settings.cpp index e2ee0e680c..411388bdb7 100644 --- a/ydb/library/yql/providers/yt/common/yql_yt_settings.cpp +++ b/ydb/library/yql/providers/yt/common/yql_yt_settings.cpp @@ -451,7 +451,6 @@ TYtConfiguration::TYtConfiguration() } return res; }); - REGISTER_SETTING(*this, CostBasedOptimizer).Parser([](const TString& v) { return FromString<ECostBasedOptimizer>(v); }); } EReleaseTempDataMode GetReleaseTempDataMode(const TYtSettings& settings) { diff --git a/ydb/library/yql/providers/yt/common/yql_yt_settings.h b/ydb/library/yql/providers/yt/common/yql_yt_settings.h index cbb59d71ee..f688c78533 100644 --- a/ydb/library/yql/providers/yt/common/yql_yt_settings.h +++ b/ydb/library/yql/providers/yt/common/yql_yt_settings.h @@ -58,12 +58,6 @@ enum class EStoredConstraint : ui64 { Distinct = 4ULL /* "Distinct" */, }; -enum class ECostBasedOptimizer { - Disable, - PG, - Native, -}; - enum class EInferSchemaMode { Sequential = 0ULL /* "sequential" */, Parallel = 1ULL /* "parallel" */, @@ -266,7 +260,6 @@ struct TYtSettings { NCommon::TConfSetting<double, false> MaxCpuUsageToFuseMultiOuts; NCommon::TConfSetting<double, false> MaxReplicationFactorToFuseMultiOuts; NCommon::TConfSetting<ui64, false> ApplyStoredConstraints; - NCommon::TConfSetting<ECostBasedOptimizer, false> CostBasedOptimizer; }; EReleaseTempDataMode GetReleaseTempDataMode(const TYtSettings& settings); diff --git a/ydb/library/yql/providers/yt/provider/ut/yql_yt_cbo_ut.cpp b/ydb/library/yql/providers/yt/provider/ut/yql_yt_cbo_ut.cpp index 109c633610..6fe93acf9e 100644 --- a/ydb/library/yql/providers/yt/provider/ut/yql_yt_cbo_ut.cpp +++ b/ydb/library/yql/providers/yt/provider/ut/yql_yt_cbo_ut.cpp @@ -61,7 +61,8 @@ Y_UNIT_TEST(OrderJoinsDoesNothingWhenCBODisabled) { TYtState::TPtr state = MakeIntrusive<TYtState>(); TYtJoinNodeOp::TPtr tree = nullptr; TYtJoinNodeOp::TPtr optimizedTree; - state->Configuration->CostBasedOptimizer = ECostBasedOptimizer::Disable; + TTypeAnnotationContext typeCtx; + state->Types = &typeCtx; TExprContext ctx; @@ -71,21 +72,23 @@ Y_UNIT_TEST(OrderJoinsDoesNothingWhenCBODisabled) { #define ADD_TEST(Name) \ Y_UNIT_TEST(Name ## _PG) { \ - Name(ECostBasedOptimizer::PG); \ + Name(ECostBasedOptimizerType::PG); \ } \ Y_UNIT_TEST(Name ## _Native) { \ - Name(ECostBasedOptimizer::Native); \ + Name(ECostBasedOptimizerType::Native); \ } -void OrderJoins2Tables(ECostBasedOptimizer optimizerType) { +void OrderJoins2Tables(auto optimizerType) { TExprContext exprCtx; auto tree = MakeOp({"c", "c_nationkey"}, {"n", "n_nationkey"}, {"c", "n"}, exprCtx); tree->Left = MakeLeaf({"c"}, {"c"}, 100000, 12333, exprCtx); tree->Right = MakeLeaf({"n"}, {"n"}, 1000, 1233, exprCtx); TYtState::TPtr state = MakeIntrusive<TYtState>(); - state->Configuration->CostBasedOptimizer = optimizerType; + TTypeAnnotationContext typeCtx; + typeCtx.CostBasedOptimizer = optimizerType; + state->Types = &typeCtx; auto optimizedTree = OrderJoins(tree, state, exprCtx, true); UNIT_ASSERT(optimizedTree != tree); UNIT_ASSERT(optimizedTree->Left); @@ -103,30 +106,34 @@ void OrderJoins2Tables(ECostBasedOptimizer optimizerType) { ADD_TEST(OrderJoins2Tables) -void OrderJoins2TablesComplexLabel(ECostBasedOptimizer optimizerType) +void OrderJoins2TablesComplexLabel(auto optimizerType) { TExprContext exprCtx; auto tree = MakeOp({"c", "c_nationkey"}, {"n", "n_nationkey"}, {"c", "n", "e"}, exprCtx); tree->Left = MakeLeaf({"c"}, {"c"}, 1000000, 1233333, exprCtx); tree->Right = MakeLeaf({"n"}, {"n", "e"}, 10000, 12333, exprCtx); + TTypeAnnotationContext typeCtx; TYtState::TPtr state = MakeIntrusive<TYtState>(); - state->Configuration->CostBasedOptimizer = optimizerType; + typeCtx.CostBasedOptimizer = optimizerType; + state->Types = &typeCtx; auto optimizedTree = OrderJoins(tree, state, exprCtx, true); UNIT_ASSERT(optimizedTree != tree); } ADD_TEST(OrderJoins2TablesComplexLabel) -void OrderJoins2TablesTableIn2Rels(ECostBasedOptimizer optimizerType) +void OrderJoins2TablesTableIn2Rels(auto optimizerType) { TExprContext exprCtx; auto tree = MakeOp({"c", "c_nationkey"}, {"n", "n_nationkey"}, {"c", "n", "e"}, exprCtx); tree->Left = MakeLeaf({"c"}, {"c"}, 1000000, 1233333, exprCtx); tree->Right = MakeLeaf({"n"}, {"n", "c"}, 10000, 12333, exprCtx); + TTypeAnnotationContext typeCtx; TYtState::TPtr state = MakeIntrusive<TYtState>(); - state->Configuration->CostBasedOptimizer = optimizerType; + typeCtx.CostBasedOptimizer = optimizerType; + state->Types = &typeCtx; auto optimizedTree = OrderJoins(tree, state, exprCtx, true); UNIT_ASSERT(optimizedTree != tree); } @@ -141,8 +148,10 @@ Y_UNIT_TEST(OrderLeftJoin) tree->Right = MakeLeaf({"n"}, {"n"}, 10000, 12333, exprCtx); tree->JoinKind = exprCtx.NewAtom(exprCtx.AppendPosition({}), "Left"); + TTypeAnnotationContext typeCtx; TYtState::TPtr state = MakeIntrusive<TYtState>(); - state->Configuration->CostBasedOptimizer = ECostBasedOptimizer::PG; + typeCtx.CostBasedOptimizer = ECostBasedOptimizerType::PG; + state->Types = &typeCtx; auto optimizedTree = OrderJoins(tree, state, exprCtx, true); UNIT_ASSERT(optimizedTree != tree); UNIT_ASSERT_STRINGS_EQUAL("Left", optimizedTree->JoinKind->Content()); @@ -156,8 +165,10 @@ Y_UNIT_TEST(UnsupportedJoin) tree->Right = MakeLeaf({"n"}, {"n"}, 10000, 12333, exprCtx); tree->JoinKind = exprCtx.NewAtom(exprCtx.AppendPosition({}), "Full"); + TTypeAnnotationContext typeCtx; TYtState::TPtr state = MakeIntrusive<TYtState>(); - state->Configuration->CostBasedOptimizer = ECostBasedOptimizer::PG; + typeCtx.CostBasedOptimizer = ECostBasedOptimizerType::PG; + state->Types = &typeCtx; auto optimizedTree = OrderJoins(tree, state, exprCtx, true); UNIT_ASSERT(optimizedTree == tree); } diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_join_reorder.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_join_reorder.cpp index 73da877d51..f5f380e260 100644 --- a/ydb/library/yql/providers/yt/provider/yql_yt_join_reorder.cpp +++ b/ydb/library/yql/providers/yt/provider/yql_yt_join_reorder.cpp @@ -76,12 +76,10 @@ public: TYtJoinNodeOp::TPtr op, const TYtState::TPtr& state, TExprContext& ctx, - ECostBasedOptimizer optimizerType, bool debug = false) : Root(op) , State(state) , Ctx(ctx) - , OptimizerType(optimizerType) , Debug(debug) { Y_UNUSED(State); @@ -113,17 +111,16 @@ public: std::unique_ptr<IOptimizer> opt; - switch (OptimizerType) { - case ECostBasedOptimizer::PG: + switch (State->Types->CostBasedOptimizer) { + case ECostBasedOptimizerType::PG: opt = std::unique_ptr<IOptimizer>(MakePgOptimizer(input, log)); break; - case ECostBasedOptimizer::Native: + case ECostBasedOptimizerType::Native: opt = std::unique_ptr<IOptimizer>(NDq::MakeNativeOptimizer(input, log)); break; default: - YQL_CLOG(ERROR, ProviderYt) << "Unknown optimizer type"; + YQL_CLOG(ERROR, ProviderYt) << "Unknown optimizer type " << ToString(State->Types->CostBasedOptimizer); return Root; - break; } try { @@ -396,7 +393,6 @@ private: TYtJoinNodeOp::TPtr Root; const TYtState::TPtr& State; TExprContext& Ctx; - ECostBasedOptimizer OptimizerType; bool Debug; THashMap<TStringBuf, std::vector<int>> Table2RelIds; @@ -418,12 +414,11 @@ private: TYtJoinNodeOp::TPtr OrderJoins(TYtJoinNodeOp::TPtr op, const TYtState::TPtr& state, TExprContext& ctx, bool debug) { - auto optimizerType = state->Configuration->CostBasedOptimizer.Get().GetOrElse(ECostBasedOptimizer::Disable); - if (optimizerType == ECostBasedOptimizer::Disable) { + if (state->Types->CostBasedOptimizer == ECostBasedOptimizerType::Disable) { return op; } - auto result = TJoinReorderer(op, state, ctx, optimizerType, debug).Do(); + auto result = TJoinReorderer(op, state, ctx, debug).Do(); if (!debug && AreSimilarTrees(result, op)) { return op; } diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_physical_optimize.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_physical_optimize.cpp index 503dfd69ac..ffeda04f2f 100644 --- a/ydb/library/yql/providers/yt/provider/yql_yt_physical_optimize.cpp +++ b/ydb/library/yql/providers/yt/provider/yql_yt_physical_optimize.cpp @@ -5599,7 +5599,7 @@ private: TMaybeNode<TExprBase> RuntimeEquiJoin(TExprBase node, TExprContext& ctx) const { auto equiJoin = node.Cast<TYtEquiJoin>(); - const bool tryReorder = State_->Configuration->CostBasedOptimizer.Get().GetOrElse(ECostBasedOptimizer::Disable) != ECostBasedOptimizer::Disable + const bool tryReorder = State_->Types->CostBasedOptimizer != ECostBasedOptimizerType::Disable && equiJoin.Input().Size() > 2 && HasOnlyOneJoinType(*equiJoin.Joins().Ptr(), "Inner"); diff --git a/ydb/library/yql/tests/sql/suites/join/join_cbo_3_tables.sql b/ydb/library/yql/tests/sql/suites/join/join_cbo_3_tables.sql index 8427c77d2d..12d53dd38f 100644 --- a/ydb/library/yql/tests/sql/suites/join/join_cbo_3_tables.sql +++ b/ydb/library/yql/tests/sql/suites/join/join_cbo_3_tables.sql @@ -1,6 +1,6 @@ use plato; -pragma yt.CostBasedOptimizer="PG"; +pragma CostBasedOptimizer="pg"; select i1.value, i2.value, i3.value, i4.value from Input1 as i1 diff --git a/ydb/library/yql/tests/sql/suites/join/join_left_cbo.sql b/ydb/library/yql/tests/sql/suites/join/join_left_cbo.sql index 8c3b6edacd..81b974bdce 100644 --- a/ydb/library/yql/tests/sql/suites/join/join_left_cbo.sql +++ b/ydb/library/yql/tests/sql/suites/join/join_left_cbo.sql @@ -1,6 +1,6 @@ use plato; -pragma yt.CostBasedOptimizer="PG"; +pragma CostBasedOptimizer="PG"; select i1.value, i2.value from Input1 as i1 diff --git a/ydb/library/yql/tests/sql/suites/join/join_right_cbo.sql b/ydb/library/yql/tests/sql/suites/join/join_right_cbo.sql index fdcad9f79c..7b9378a9ec 100644 --- a/ydb/library/yql/tests/sql/suites/join/join_right_cbo.sql +++ b/ydb/library/yql/tests/sql/suites/join/join_right_cbo.sql @@ -1,6 +1,6 @@ use plato; -pragma yt.CostBasedOptimizer="PG"; +pragma CostBasedOptimizer="PG"; select i1.value, i2.value from Input1 as i1 |