summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAndrey Neporada <[email protected]>2024-01-19 18:05:30 +0300
committerGitHub <[email protected]>2024-01-19 18:05:30 +0300
commit608c077c3d94e0a2af4d69634aed61405ad161f6 (patch)
tree5bf3cb824467e54f940c49de6b3c56c43843c084
parent56e9a2180c82c7a8a24cf6c00b2f56578683eb4b (diff)
[YQL-17474] Limit supported operation settings in hybrid execution (#1132)
* [YQL-17474] Limit supported operation settings in hybrid execution * update supported settings list
-rw-r--r--ydb/library/yql/providers/yt/provider/yql_yt_dq_hybrid.cpp4
-rw-r--r--ydb/library/yql/providers/yt/provider/yql_yt_op_settings.h34
2 files changed, 22 insertions, 16 deletions
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_dq_hybrid.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_dq_hybrid.cpp
index a97cdb90180..2c611b1d391 100644
--- a/ydb/library/yql/providers/yt/provider/yql_yt_dq_hybrid.cpp
+++ b/ydb/library/yql/providers/yt/provider/yql_yt_dq_hybrid.cpp
@@ -76,7 +76,7 @@ private:
if (const auto& trans = operation.Maybe<TYtTransientOpBase>(); trans && trans.Cast().Input().Size() != 1U)
return false;
- return !HasSetting(*operation.Ref().Child(4U), EYtSettingType::NoDq);
+ return !HasSettingsExcept(*operation.Ref().Child(4U), DqOpSupportedSettings);
}
std::optional<std::array<ui64, 2U>> CanReadHybrid(const TYtSection& section) const {
@@ -289,7 +289,7 @@ private:
}
TMaybeNode<TExprBase> TryYtMergeByDq(TExprBase node, TExprContext& ctx) const {
- if (const auto merge = node.Cast<TYtMerge>(); CanReplaceOnHybrid(merge) && !NYql::HasSetting(merge.Settings().Ref(), EYtSettingType::CombineChunks)) {
+ if (const auto merge = node.Cast<TYtMerge>(); CanReplaceOnHybrid(merge)) {
if (const auto sizeLimit = State_->Configuration->HybridDqDataSizeLimitForOrdered.Get().GetOrElse(DefaultHybridDqDataSizeLimitForOrdered)) {
const auto info = TYtTableBaseInfo::Parse(merge.Input().Item(0).Paths().Item(0).Table());
const auto chunksLimit = State_->Configuration->MaxChunksForDqRead.Get().GetOrElse(DEFAULT_MAX_CHUNKS_FOR_DQ_READ);
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_op_settings.h b/ydb/library/yql/providers/yt/provider/yql_yt_op_settings.h
index c0a27d1ec3b..846c0f19fb8 100644
--- a/ydb/library/yql/providers/yt/provider/yql_yt_op_settings.h
+++ b/ydb/library/yql/providers/yt/provider/yql_yt_op_settings.h
@@ -73,32 +73,34 @@ enum class EYtSettingType: ui64 {
ItemsCount = 1ull << 25 /* "itemsCount" */,
RowFactor = 1ull << 26 /* "rowFactor" */,
// Operations
- Ordered = 1ull << 27 /* "ordered" */,
+ Ordered = 1ull << 27 /* "ordered" */, // hybrid supported
KeyFilter = 1ull << 28 /* "keyFilter" */,
KeyFilter2 = 1ull << 29 /* "keyFilter2" */,
Take = 1ull << 30 /* "take" */,
Skip = 1ull << 31 /* "skip" */,
- Limit = 1ull << 32 /* "limit" */,
- SortLimitBy = 1ull << 33 /* "sortLimitBy" */,
- SortBy = 1ull << 34 /* "sortBy" */,
- ReduceBy = 1ull << 35 /* "reduceBy" */,
+ Limit = 1ull << 32 /* "limit" */, // hybrid supported
+ SortLimitBy = 1ull << 33 /* "sortLimitBy" */, // hybrid supported
+ SortBy = 1ull << 34 /* "sortBy" */, // hybrid supported
+ ReduceBy = 1ull << 35 /* "reduceBy" */, // hybrid supported
ReduceFilterBy = 1ull << 36 /* "reduceFilterBy" */,
- ForceTransform = 1ull << 37 /* "forceTransform" */,
+ ForceTransform = 1ull << 37 /* "forceTransform" */, // hybrid supported
WeakFields = 1ull << 38 /* "weakFields" */,
Sharded = 1ull << 39 /* "sharded" */,
CombineChunks = 1ull << 40 /* "combineChunks" */,
- JobCount = 1ull << 41 /* "jobCount" */,
- JoinReduce = 1ull << 42 /* "joinReduce" */,
- FirstAsPrimary = 1ull << 43 /* "firstAsPrimary" */,
- Flow = 1ull << 44 /* "flow" */,
- KeepSorted = 1ull << 45 /* "keepSorted" */,
- KeySwitch = 1ull << 46 /* "keySwitch" */,
+ JobCount = 1ull << 41 /* "jobCount" */, // hybrid supported
+ JoinReduce = 1ull << 42 /* "joinReduce" */, // hybrid supported
+ FirstAsPrimary = 1ull << 43 /* "firstAsPrimary" */, // hybrid supported
+ Flow = 1ull << 44 /* "flow" */, // hybrid supported
+ KeepSorted = 1ull << 45 /* "keepSorted" */, // hybrid supported
+ KeySwitch = 1ull << 46 /* "keySwitch" */, // hybrid supported
// Out tables
UniqueBy = 1ull << 47 /* "uniqueBy" */,
OpHash = 1ull << 48 /* "opHash" */,
- MapOutputType = 1ull << 49 /* "mapOutputType" */,
- ReduceInputType = 1ull << 50 /* "reduceInputType" */,
+ // Operations
+ MapOutputType = 1ull << 49 /* "mapOutputType" */, // hybrid supported
+ ReduceInputType = 1ull << 50 /* "reduceInputType" */, // hybrid supported
NoDq = 1ull << 51 /* "noDq" */,
+ // Read
Split = 1ull << 52 /* "split" */,
// Write hints
CompressionCodec = 1ull << 53 /* "compression_codec" "compressioncodec"*/,
@@ -116,6 +118,10 @@ Y_DECLARE_FLAGS(EYtSettingTypes, EYtSettingType);
Y_DECLARE_OPERATORS_FOR_FLAGS(EYtSettingTypes);
constexpr auto DqReadSupportedSettings = EYtSettingType::SysColumns | EYtSettingType::Sample | EYtSettingType::Unordered | EYtSettingType::NonUnique;
+constexpr auto DqOpSupportedSettings = EYtSettingType::Ordered | EYtSettingType::Limit | EYtSettingType::SortLimitBy | EYtSettingType::SortBy |
+ EYtSettingType::ReduceBy | EYtSettingType::ForceTransform | EYtSettingType::JobCount | EYtSettingType::JoinReduce |
+ EYtSettingType::FirstAsPrimary | EYtSettingType::Flow | EYtSettingType::KeepSorted | EYtSettingType::KeySwitch |
+ EYtSettingType::ReduceInputType | EYtSettingType::MapOutputType;
///////////////////////////////////////////////////////////////////////////////////////////////