aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorVitalii Gridnev <gridnevvvit@gmail.com>2024-07-18 19:42:59 +0400
committerGitHub <noreply@github.com>2024-07-18 19:42:59 +0400
commit6de734054b9e3217fd9d997921cfb0cf9492f0bb (patch)
treeee5725aeae1a0cdc3693ce28f87fb1b9d9a40f14
parent31ea815dbf71fb690ebea594e04097d0f221e56d (diff)
downloadydb-6de734054b9e3217fd9d997921cfb0cf9492f0bb.tar.gz
add spilling nodes and pragma (#6769)
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_exec.cpp3
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_settings.cpp21
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_settings.h2
-rw-r--r--ydb/core/kqp/query_compiler/kqp_query_compiler.cpp3
-rw-r--r--ydb/library/yql/providers/dq/common/yql_dq_settings.cpp2
-rw-r--r--ydb/library/yql/providers/dq/common/yql_dq_settings.h1
6 files changed, 29 insertions, 3 deletions
diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp
index a1c449d48b..9fc7b8d0c4 100644
--- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp
@@ -765,10 +765,11 @@ private:
TVector<TExprBase> fakeReads;
auto paramsType = NDq::CollectParameters(programLambda, ctx);
+ NDq::TSpillingSettings spillingSettings{SessionCtx->Config().GetEnabledSpillingNodes()};
lambda = NDq::BuildProgram(
programLambda, *paramsType, compiler, SessionCtx->Query().QueryData->GetAllocState()->TypeEnv,
*SessionCtx->Query().QueryData->GetAllocState()->HolderFactory.GetFunctionRegistry(),
- ctx, fakeReads, {});
+ ctx, fakeReads, spillingSettings);
NKikimr::NMiniKQL::TProgramBuilder programBuilder(SessionCtx->Query().QueryData->GetAllocState()->TypeEnv,
*SessionCtx->Query().QueryData->GetAllocState()->HolderFactory.GetFunctionRegistry());
diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.cpp b/ydb/core/kqp/provider/yql_kikimr_settings.cpp
index 79d50d96f2..0f2cd04f09 100644
--- a/ydb/core/kqp/provider/yql_kikimr_settings.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_settings.cpp
@@ -3,6 +3,8 @@
#include <ydb/core/protos/config.pb.h>
#include <ydb/core/protos/table_service_config.pb.h>
#include <util/generic/size_literals.h>
+#include <util/string/split.h>
+#include <ydb/library/yql/providers/dq/common/yql_dq_settings.h>
namespace NYql {
@@ -69,6 +71,21 @@ TKikimrConfiguration::TKikimrConfiguration() {
REGISTER_SETTING(*this, OptUseFinalizeByKey);
REGISTER_SETTING(*this, CostBasedOptimizationLevel);
+ REGISTER_SETTING(*this, EnableSpillingNodes)
+ .Parser([](const TString& v) {
+ ui64 res = 0;
+ TVector<TString> vec;
+ StringSplitter(v).SplitBySet(",;| ").AddTo(&vec);
+ for (auto& s: vec) {
+ if (s.empty()) {
+ throw yexception() << "Empty value item";
+ }
+ auto value = FromStringWithDefault<NYql::TDqSettings::EEnabledSpillingNodes>(
+ s, NYql::TDqSettings::EEnabledSpillingNodes::None);
+ res |= ui64(value);
+ }
+ return res;
+ });
REGISTER_SETTING(*this, MaxDPccpDPTableSize);
@@ -125,6 +142,10 @@ bool TKikimrSettings::HasOptUseFinalizeByKey() const {
return GetOptionalFlagValue(OptUseFinalizeByKey.Get()) != EOptionalFlag::Disabled;
}
+ui64 TKikimrSettings::GetEnabledSpillingNodes() const {
+ return EnableSpillingNodes.Get().GetOrElse(0);
+}
+
EOptionalFlag TKikimrSettings::GetOptPredicateExtract() const {
return GetOptionalFlagValue(OptEnablePredicateExtract.Get());
diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.h b/ydb/core/kqp/provider/yql_kikimr_settings.h
index bb57a76079..cf4757d9a3 100644
--- a/ydb/core/kqp/provider/yql_kikimr_settings.h
+++ b/ydb/core/kqp/provider/yql_kikimr_settings.h
@@ -51,6 +51,7 @@ struct TKikimrSettings {
NCommon::TConfSetting<bool, false> EnableLlvm;
NCommon::TConfSetting<NDq::EHashJoinMode, false> HashJoinMode;
NCommon::TConfSetting<TString, false> OverrideStatistics;
+ NCommon::TConfSetting<ui64, false> EnableSpillingNodes;
/* Disable optimizer rules */
NCommon::TConfSetting<bool, false> OptDisableTopSort;
@@ -82,6 +83,7 @@ struct TKikimrSettings {
bool HasOptEnableOlapPushdown() const;
bool HasOptEnableOlapProvideComputeSharding() const;
bool HasOptUseFinalizeByKey() const;
+ ui64 GetEnabledSpillingNodes() const;
EOptionalFlag GetOptPredicateExtract() const;
EOptionalFlag GetUseLlvm() const;
diff --git a/ydb/core/kqp/query_compiler/kqp_query_compiler.cpp b/ydb/core/kqp/query_compiler/kqp_query_compiler.cpp
index a3087c2181..15f65fa957 100644
--- a/ydb/core/kqp/query_compiler/kqp_query_compiler.cpp
+++ b/ydb/core/kqp/query_compiler/kqp_query_compiler.cpp
@@ -777,8 +777,9 @@ private:
stageProto.SetIsEffectsStage(hasEffects || hasTxTableSink);
auto paramsType = CollectParameters(stage, ctx);
+ NDq::TSpillingSettings spillingSettings{Config->GetEnabledSpillingNodes()};
auto programBytecode = NDq::BuildProgram(stage.Program(), *paramsType, *KqlCompiler, TypeEnv, FuncRegistry,
- ctx, {}, {});
+ ctx, {}, spillingSettings);
auto& programProto = *stageProto.MutableProgram();
programProto.SetRuntimeVersion(NYql::NDqProto::ERuntimeVersion::RUNTIME_VERSION_YQL_1_0);
diff --git a/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp b/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp
index 24f16bd283..996a7cd1ae 100644
--- a/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp
+++ b/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp
@@ -110,7 +110,7 @@ TDqConfiguration::TDqConfiguration() {
if (s.empty()) {
throw yexception() << "Empty value item";
}
- auto value = FromString<EEnabledSpillingNodes>(s);
+ auto value = FromStringWithDefault<EEnabledSpillingNodes>(s, EEnabledSpillingNodes::None);
res |= ui64(value);
}
return res;
diff --git a/ydb/library/yql/providers/dq/common/yql_dq_settings.h b/ydb/library/yql/providers/dq/common/yql_dq_settings.h
index 1b9c63dd6c..8fe9163d90 100644
--- a/ydb/library/yql/providers/dq/common/yql_dq_settings.h
+++ b/ydb/library/yql/providers/dq/common/yql_dq_settings.h
@@ -29,6 +29,7 @@ struct TDqSettings {
};
enum class EEnabledSpillingNodes : ui64 {
+ None = 0ULL /* None */,
GraceJoin = 1ULL /* "GraceJoin" */,
All = ~0ULL /* "All" */,
};