diff options
| author | Andrey Neporada <[email protected]> | 2024-01-19 18:05:30 +0300 |
|---|---|---|
| committer | GitHub <[email protected]> | 2024-01-19 18:05:30 +0300 |
| commit | 608c077c3d94e0a2af4d69634aed61405ad161f6 (patch) | |
| tree | 5bf3cb824467e54f940c49de6b3c56c43843c084 | |
| parent | 56e9a2180c82c7a8a24cf6c00b2f56578683eb4b (diff) | |
[YQL-17474] Limit supported operation settings in hybrid execution (#1132)
* [YQL-17474] Limit supported operation settings in hybrid execution
* update supported settings list
| -rw-r--r-- | ydb/library/yql/providers/yt/provider/yql_yt_dq_hybrid.cpp | 4 | ||||
| -rw-r--r-- | ydb/library/yql/providers/yt/provider/yql_yt_op_settings.h | 34 |
2 files changed, 22 insertions, 16 deletions
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_dq_hybrid.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_dq_hybrid.cpp index a97cdb90180..2c611b1d391 100644 --- a/ydb/library/yql/providers/yt/provider/yql_yt_dq_hybrid.cpp +++ b/ydb/library/yql/providers/yt/provider/yql_yt_dq_hybrid.cpp @@ -76,7 +76,7 @@ private: if (const auto& trans = operation.Maybe<TYtTransientOpBase>(); trans && trans.Cast().Input().Size() != 1U) return false; - return !HasSetting(*operation.Ref().Child(4U), EYtSettingType::NoDq); + return !HasSettingsExcept(*operation.Ref().Child(4U), DqOpSupportedSettings); } std::optional<std::array<ui64, 2U>> CanReadHybrid(const TYtSection& section) const { @@ -289,7 +289,7 @@ private: } TMaybeNode<TExprBase> TryYtMergeByDq(TExprBase node, TExprContext& ctx) const { - if (const auto merge = node.Cast<TYtMerge>(); CanReplaceOnHybrid(merge) && !NYql::HasSetting(merge.Settings().Ref(), EYtSettingType::CombineChunks)) { + if (const auto merge = node.Cast<TYtMerge>(); CanReplaceOnHybrid(merge)) { if (const auto sizeLimit = State_->Configuration->HybridDqDataSizeLimitForOrdered.Get().GetOrElse(DefaultHybridDqDataSizeLimitForOrdered)) { const auto info = TYtTableBaseInfo::Parse(merge.Input().Item(0).Paths().Item(0).Table()); const auto chunksLimit = State_->Configuration->MaxChunksForDqRead.Get().GetOrElse(DEFAULT_MAX_CHUNKS_FOR_DQ_READ); diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_op_settings.h b/ydb/library/yql/providers/yt/provider/yql_yt_op_settings.h index c0a27d1ec3b..846c0f19fb8 100644 --- a/ydb/library/yql/providers/yt/provider/yql_yt_op_settings.h +++ b/ydb/library/yql/providers/yt/provider/yql_yt_op_settings.h @@ -73,32 +73,34 @@ enum class EYtSettingType: ui64 { ItemsCount = 1ull << 25 /* "itemsCount" */, RowFactor = 1ull << 26 /* "rowFactor" */, // Operations - Ordered = 1ull << 27 /* "ordered" */, + Ordered = 1ull << 27 /* "ordered" */, // hybrid supported KeyFilter = 1ull << 28 /* "keyFilter" */, KeyFilter2 = 1ull << 29 /* "keyFilter2" */, Take = 1ull << 30 /* "take" */, Skip = 1ull << 31 /* "skip" */, - Limit = 1ull << 32 /* "limit" */, - SortLimitBy = 1ull << 33 /* "sortLimitBy" */, - SortBy = 1ull << 34 /* "sortBy" */, - ReduceBy = 1ull << 35 /* "reduceBy" */, + Limit = 1ull << 32 /* "limit" */, // hybrid supported + SortLimitBy = 1ull << 33 /* "sortLimitBy" */, // hybrid supported + SortBy = 1ull << 34 /* "sortBy" */, // hybrid supported + ReduceBy = 1ull << 35 /* "reduceBy" */, // hybrid supported ReduceFilterBy = 1ull << 36 /* "reduceFilterBy" */, - ForceTransform = 1ull << 37 /* "forceTransform" */, + ForceTransform = 1ull << 37 /* "forceTransform" */, // hybrid supported WeakFields = 1ull << 38 /* "weakFields" */, Sharded = 1ull << 39 /* "sharded" */, CombineChunks = 1ull << 40 /* "combineChunks" */, - JobCount = 1ull << 41 /* "jobCount" */, - JoinReduce = 1ull << 42 /* "joinReduce" */, - FirstAsPrimary = 1ull << 43 /* "firstAsPrimary" */, - Flow = 1ull << 44 /* "flow" */, - KeepSorted = 1ull << 45 /* "keepSorted" */, - KeySwitch = 1ull << 46 /* "keySwitch" */, + JobCount = 1ull << 41 /* "jobCount" */, // hybrid supported + JoinReduce = 1ull << 42 /* "joinReduce" */, // hybrid supported + FirstAsPrimary = 1ull << 43 /* "firstAsPrimary" */, // hybrid supported + Flow = 1ull << 44 /* "flow" */, // hybrid supported + KeepSorted = 1ull << 45 /* "keepSorted" */, // hybrid supported + KeySwitch = 1ull << 46 /* "keySwitch" */, // hybrid supported // Out tables UniqueBy = 1ull << 47 /* "uniqueBy" */, OpHash = 1ull << 48 /* "opHash" */, - MapOutputType = 1ull << 49 /* "mapOutputType" */, - ReduceInputType = 1ull << 50 /* "reduceInputType" */, + // Operations + MapOutputType = 1ull << 49 /* "mapOutputType" */, // hybrid supported + ReduceInputType = 1ull << 50 /* "reduceInputType" */, // hybrid supported NoDq = 1ull << 51 /* "noDq" */, + // Read Split = 1ull << 52 /* "split" */, // Write hints CompressionCodec = 1ull << 53 /* "compression_codec" "compressioncodec"*/, @@ -116,6 +118,10 @@ Y_DECLARE_FLAGS(EYtSettingTypes, EYtSettingType); Y_DECLARE_OPERATORS_FOR_FLAGS(EYtSettingTypes); constexpr auto DqReadSupportedSettings = EYtSettingType::SysColumns | EYtSettingType::Sample | EYtSettingType::Unordered | EYtSettingType::NonUnique; +constexpr auto DqOpSupportedSettings = EYtSettingType::Ordered | EYtSettingType::Limit | EYtSettingType::SortLimitBy | EYtSettingType::SortBy | + EYtSettingType::ReduceBy | EYtSettingType::ForceTransform | EYtSettingType::JobCount | EYtSettingType::JoinReduce | + EYtSettingType::FirstAsPrimary | EYtSettingType::Flow | EYtSettingType::KeepSorted | EYtSettingType::KeySwitch | + EYtSettingType::ReduceInputType | EYtSettingType::MapOutputType; /////////////////////////////////////////////////////////////////////////////////////////////// |
