diff options
| author | udovichenko-r <udovichenko-r@yandex-team.com> | 2025-01-27 13:14:24 +0300 |
|---|---|---|
| committer | udovichenko-r <udovichenko-r@yandex-team.com> | 2025-01-27 13:34:02 +0300 |
| commit | dbe0a2136c5f26060cb19a426c7cf798aad8e423 (patch) | |
| tree | b1bba5b09f14d17161bc2c447d4f67436c54d6c6 | |
| parent | b6be683d22881e476059e543ea0ada650219f4d0 (diff) | |
| download | ydb-dbe0a2136c5f26060cb19a426c7cf798aad8e423.tar.gz | |
Drop keepSorted when fuse YtMapReduce with unsorted YtMap
commit_hash:3778c34728349609722faef1a43561086c814f4f
3 files changed, 21 insertions, 1 deletions
diff --git a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp index 821666afe4..baead6e7ec 100644 --- a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp +++ b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp @@ -877,7 +877,12 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::FuseOuterMap(TExprBase lambda.Ptr()); res = ctx.ChangeChild(*res, TYtWithUserJobsOpBase::idx_Output, outerMap.Output().Ptr()); - auto mergedSettings = NYql::RemoveSettings(outerMap.Settings().Ref(), EYtSettingType::Ordered | EYtSettingType::Sharded | EYtSettingType::Flow | EYtSettingType::BlockInputReady | EYtSettingType::BlockOutputReady, ctx); + EYtSettingTypes toRemove = EYtSettingType::Ordered | EYtSettingType::Sharded | EYtSettingType::Flow | EYtSettingType::BlockInputReady | EYtSettingType::BlockOutputReady; + if (inner.Maybe<TYtMapReduce>()) { + // Can be safely removed, because outer map has no sorted outputs (checked below) + toRemove |= EYtSettingType::KeepSorted; + } + auto mergedSettings = NYql::RemoveSettings(outerMap.Settings().Ref(), toRemove, ctx); mergedSettings = MergeSettings(inner.Settings().Ref(), *mergedSettings, ctx); res = ctx.ChangeChild(*res, TYtWithUserJobsOpBase::idx_Settings, std::move(mergedSettings)); res = ctx.ChangeChild(*res, TYtWithUserJobsOpBase::idx_World, diff --git a/yt/yql/tests/sql/suites/optimizers/remove_keep_sorted_setting.cfg b/yt/yql/tests/sql/suites/optimizers/remove_keep_sorted_setting.cfg new file mode 100644 index 0000000000..eb8f24f034 --- /dev/null +++ b/yt/yql/tests/sql/suites/optimizers/remove_keep_sorted_setting.cfg @@ -0,0 +1,2 @@ +in Input1 sorted.txt +in Input2 sorted.txt diff --git a/yt/yql/tests/sql/suites/optimizers/remove_keep_sorted_setting.sql b/yt/yql/tests/sql/suites/optimizers/remove_keep_sorted_setting.sql new file mode 100644 index 0000000000..4e6be632db --- /dev/null +++ b/yt/yql/tests/sql/suites/optimizers/remove_keep_sorted_setting.sql @@ -0,0 +1,13 @@ +USE plato; + +$i = SELECT + a.key as key, + b.subkey as subkey +FROM Input1 AS a +JOIN Input2 AS b ON a.key = b.key +ASSUME ORDER BY key, subkey; + +$i = SELECT * FROM $i WHERE key != subkey ASSUME ORDER BY key, subkey; + +SELECT DISTINCT key, subkey FROM $i; +SELECT COUNT(*) FROM $i; |