aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorspuchin <spuchin@ydb.tech>2023-03-13 14:14:38 +0300
committerspuchin <spuchin@ydb.tech>2023-03-13 14:14:38 +0300
commitd35f75d44da44fafcf19f2699559286d29ff6790 (patch)
treef5c042046513f7cfe03309e42e4fbb3f88917c1f
parentdb11197d6262105fddc5d10bd6f64d2441b55ed2 (diff)
downloadydb-d35f75d44da44fafcf19f2699559286d29ff6790.tar.gz
Remove legacy DqStage settings. ()
-rw-r--r--ydb/library/yql/dq/opt/dq_opt.cpp18
-rw-r--r--ydb/library/yql/dq/opt/dq_opt.h15
-rw-r--r--ydb/library/yql/providers/dq/planner/execution_planner.cpp3
3 files changed, 2 insertions, 34 deletions
diff --git a/ydb/library/yql/dq/opt/dq_opt.cpp b/ydb/library/yql/dq/opt/dq_opt.cpp
index 8cffde696a..bd8bfd6abe 100644
--- a/ydb/library/yql/dq/opt/dq_opt.cpp
+++ b/ydb/library/yql/dq/opt/dq_opt.cpp
@@ -21,20 +21,6 @@ TDqStageSettings TDqStageSettings::Parse(const TDqStageBase& node) {
settings.LogicalId = FromString<ui64>(tuple.Value().Cast<TCoAtom>().Value());
} else if (name == SinglePartitionSettingName) {
settings.SinglePartition = true;
- } else if (name == IsExternalSetting) {
- settings.IsExternalFunction = true;
- } else if (name == TransformNameSetting) {
- YQL_ENSURE(tuple.Value().Maybe<TCoAtom>());
- settings.TransformName = tuple.Value().Cast<TCoAtom>().Value();
- } else if (name == TransformTypeSetting) {
- YQL_ENSURE(tuple.Value().Maybe<TCoAtom>());
- if (const auto type = tuple.Value().Cast<TCoAtom>().Value(); type == "YANDEX-CLOUD") {
- settings.TransformType = "YandexCloudFunction";
- } else {
- YQL_ENSURE(false, "Unknown transform type: " << type);
- }
- } else if (name == TransformConcurrencySetting) {
- settings.TransformConcurrency = FromString<ui32>(tuple.Value().Cast<TCoAtom>().Value());
}
}
@@ -81,10 +67,6 @@ NNodes::TCoNameValueTupleList TDqStageSettings::BuildNode(TExprContext& ctx, TPo
.Done();
}
-ui32 TDqStageSettings::MaxTransformConcurrency() const {
- return TransformConcurrency > 0 ? TransformConcurrency : TDqSettings::TDefault::CloudFunctionConcurrency;
-}
-
TCoAtom BuildAtom(TStringBuf value, TPositionHandle pos, TExprContext& ctx) {
return Build<TCoAtom>(ctx, pos)
.Value(value)
diff --git a/ydb/library/yql/dq/opt/dq_opt.h b/ydb/library/yql/dq/opt/dq_opt.h
index 3887ef720a..3cb3566a2e 100644
--- a/ydb/library/yql/dq/opt/dq_opt.h
+++ b/ydb/library/yql/dq/opt/dq_opt.h
@@ -12,22 +12,11 @@ struct TDqStageSettings {
static constexpr TStringBuf LogicalIdSettingName = "_logical_id";
static constexpr TStringBuf IdSettingName = "_id";
static constexpr TStringBuf SinglePartitionSettingName = "_single_partition";
- static constexpr TStringBuf IsExternalSetting = "is_external_function";
- static constexpr TStringBuf TransformNameSetting = "transform_name";
- static constexpr TStringBuf TransformTypeSetting = "transform_type";
- static constexpr TStringBuf TransformConcurrencySetting = "concurrency";
ui64 LogicalId = 0;
TString Id;
bool SinglePartition = false;
- bool IsExternalFunction = false;
- TString TransformType;
- TString TransformName;
- ui32 TransformConcurrency = 0;
-
- static TDqStageSettings Parse(const NNodes::TDqStageBase& node);
-
static TDqStageSettings New(const NNodes::TDqStageBase& node);
static TDqStageSettings New() {
@@ -35,9 +24,9 @@ struct TDqStageSettings {
s.Id = CreateGuidAsString();
return s;
}
- NNodes::TCoNameValueTupleList BuildNode(TExprContext& ctx, TPositionHandle pos) const;
- ui32 MaxTransformConcurrency() const;
+ static TDqStageSettings Parse(const NNodes::TDqStageBase& node);
+ NNodes::TCoNameValueTupleList BuildNode(TExprContext& ctx, TPositionHandle pos) const;
};
NNodes::TCoAtom BuildAtom(TStringBuf value, TPositionHandle pos, TExprContext& ctx);
diff --git a/ydb/library/yql/providers/dq/planner/execution_planner.cpp b/ydb/library/yql/providers/dq/planner/execution_planner.cpp
index 19bb723c47..93671746fd 100644
--- a/ydb/library/yql/providers/dq/planner/execution_planner.cpp
+++ b/ydb/library/yql/providers/dq/planner/execution_planner.cpp
@@ -530,9 +530,6 @@ namespace NYql::NDqs {
YQL_ENSURE(datasource);
const auto stageSettings = TDqStageSettings::Parse(stage);
auto tasksPerStage = settings->MaxTasksPerStage.Get().GetOrElse(TDqSettings::TDefault::MaxTasksPerStage);
- if (stageSettings.IsExternalFunction) {
- tasksPerStage = Min(tasksPerStage, stageSettings.MaxTransformConcurrency());
- }
const size_t maxPartitions = stageSettings.SinglePartition ? 1ULL : tasksPerStage;
TVector<TString> parts;
if (auto dqIntegration = (*datasource)->GetDqIntegration()) {