diff options
author | spuchin <spuchin@ydb.tech> | 2023-03-13 14:14:38 +0300 |
---|---|---|
committer | spuchin <spuchin@ydb.tech> | 2023-03-13 14:14:38 +0300 |
commit | d35f75d44da44fafcf19f2699559286d29ff6790 (patch) | |
tree | f5c042046513f7cfe03309e42e4fbb3f88917c1f | |
parent | db11197d6262105fddc5d10bd6f64d2441b55ed2 (diff) | |
download | ydb-d35f75d44da44fafcf19f2699559286d29ff6790.tar.gz |
Remove legacy DqStage settings. ()
-rw-r--r-- | ydb/library/yql/dq/opt/dq_opt.cpp | 18 | ||||
-rw-r--r-- | ydb/library/yql/dq/opt/dq_opt.h | 15 | ||||
-rw-r--r-- | ydb/library/yql/providers/dq/planner/execution_planner.cpp | 3 |
3 files changed, 2 insertions, 34 deletions
diff --git a/ydb/library/yql/dq/opt/dq_opt.cpp b/ydb/library/yql/dq/opt/dq_opt.cpp index 8cffde696a..bd8bfd6abe 100644 --- a/ydb/library/yql/dq/opt/dq_opt.cpp +++ b/ydb/library/yql/dq/opt/dq_opt.cpp @@ -21,20 +21,6 @@ TDqStageSettings TDqStageSettings::Parse(const TDqStageBase& node) { settings.LogicalId = FromString<ui64>(tuple.Value().Cast<TCoAtom>().Value()); } else if (name == SinglePartitionSettingName) { settings.SinglePartition = true; - } else if (name == IsExternalSetting) { - settings.IsExternalFunction = true; - } else if (name == TransformNameSetting) { - YQL_ENSURE(tuple.Value().Maybe<TCoAtom>()); - settings.TransformName = tuple.Value().Cast<TCoAtom>().Value(); - } else if (name == TransformTypeSetting) { - YQL_ENSURE(tuple.Value().Maybe<TCoAtom>()); - if (const auto type = tuple.Value().Cast<TCoAtom>().Value(); type == "YANDEX-CLOUD") { - settings.TransformType = "YandexCloudFunction"; - } else { - YQL_ENSURE(false, "Unknown transform type: " << type); - } - } else if (name == TransformConcurrencySetting) { - settings.TransformConcurrency = FromString<ui32>(tuple.Value().Cast<TCoAtom>().Value()); } } @@ -81,10 +67,6 @@ NNodes::TCoNameValueTupleList TDqStageSettings::BuildNode(TExprContext& ctx, TPo .Done(); } -ui32 TDqStageSettings::MaxTransformConcurrency() const { - return TransformConcurrency > 0 ? TransformConcurrency : TDqSettings::TDefault::CloudFunctionConcurrency; -} - TCoAtom BuildAtom(TStringBuf value, TPositionHandle pos, TExprContext& ctx) { return Build<TCoAtom>(ctx, pos) .Value(value) diff --git a/ydb/library/yql/dq/opt/dq_opt.h b/ydb/library/yql/dq/opt/dq_opt.h index 3887ef720a..3cb3566a2e 100644 --- a/ydb/library/yql/dq/opt/dq_opt.h +++ b/ydb/library/yql/dq/opt/dq_opt.h @@ -12,22 +12,11 @@ struct TDqStageSettings { static constexpr TStringBuf LogicalIdSettingName = "_logical_id"; static constexpr TStringBuf IdSettingName = "_id"; static constexpr TStringBuf SinglePartitionSettingName = "_single_partition"; - static constexpr TStringBuf IsExternalSetting = "is_external_function"; - static constexpr TStringBuf TransformNameSetting = "transform_name"; - static constexpr TStringBuf TransformTypeSetting = "transform_type"; - static constexpr TStringBuf TransformConcurrencySetting = "concurrency"; ui64 LogicalId = 0; TString Id; bool SinglePartition = false; - bool IsExternalFunction = false; - TString TransformType; - TString TransformName; - ui32 TransformConcurrency = 0; - - static TDqStageSettings Parse(const NNodes::TDqStageBase& node); - static TDqStageSettings New(const NNodes::TDqStageBase& node); static TDqStageSettings New() { @@ -35,9 +24,9 @@ struct TDqStageSettings { s.Id = CreateGuidAsString(); return s; } - NNodes::TCoNameValueTupleList BuildNode(TExprContext& ctx, TPositionHandle pos) const; - ui32 MaxTransformConcurrency() const; + static TDqStageSettings Parse(const NNodes::TDqStageBase& node); + NNodes::TCoNameValueTupleList BuildNode(TExprContext& ctx, TPositionHandle pos) const; }; NNodes::TCoAtom BuildAtom(TStringBuf value, TPositionHandle pos, TExprContext& ctx); diff --git a/ydb/library/yql/providers/dq/planner/execution_planner.cpp b/ydb/library/yql/providers/dq/planner/execution_planner.cpp index 19bb723c47..93671746fd 100644 --- a/ydb/library/yql/providers/dq/planner/execution_planner.cpp +++ b/ydb/library/yql/providers/dq/planner/execution_planner.cpp @@ -530,9 +530,6 @@ namespace NYql::NDqs { YQL_ENSURE(datasource); const auto stageSettings = TDqStageSettings::Parse(stage); auto tasksPerStage = settings->MaxTasksPerStage.Get().GetOrElse(TDqSettings::TDefault::MaxTasksPerStage); - if (stageSettings.IsExternalFunction) { - tasksPerStage = Min(tasksPerStage, stageSettings.MaxTransformConcurrency()); - } const size_t maxPartitions = stageSettings.SinglePartition ? 1ULL : tasksPerStage; TVector<TString> parts; if (auto dqIntegration = (*datasource)->GetDqIntegration()) { |