aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorvvvv <vvvv@yandex-team.com>2025-04-21 20:53:56 +0300
committervvvv <vvvv@yandex-team.com>2025-04-21 21:26:59 +0300
commit108c1aa409626aa29ca728edafd01fbbd7a1b00e (patch)
treeb88c1e3b690a29b0ed4f505f05990d656d6373ec
parent6b17ad8c52e16d0088291781e1b5a9e28d028b9b (diff)
downloadydb-108c1aa409626aa29ca728edafd01fbbd7a1b00e.tar.gz
YQL-19861 abi, udf resolvers, test, pass via yt gateways
commit_hash:6e3f5fac6a8598586987b52d749644d1ce1fccbe
-rw-r--r--yql/essentials/core/type_ann/type_ann_core.cpp45
-rw-r--r--yql/essentials/core/type_ann/ya.make1
-rw-r--r--yql/essentials/core/yql_type_annotation.h2
-rw-r--r--yql/essentials/core/yql_udf_resolver.h4
-rw-r--r--yql/essentials/minikql/comp_nodes/mkql_scalar_apply.cpp2
-rw-r--r--yql/essentials/minikql/comp_nodes/mkql_udf.cpp8
-rw-r--r--yql/essentials/minikql/computation/mkql_computation_node.h53
-rw-r--r--yql/essentials/minikql/computation/mkql_computation_node_graph.cpp3
-rw-r--r--yql/essentials/minikql/computation/mkql_validate_ut.cpp6
-rw-r--r--yql/essentials/minikql/computation/mkql_value_builder_ut.cpp2
-rw-r--r--yql/essentials/minikql/computation/ya.make.inc1
-rw-r--r--yql/essentials/minikql/mkql_function_registry.cpp49
-rw-r--r--yql/essentials/minikql/mkql_function_registry.h24
-rw-r--r--yql/essentials/minikql/mkql_program_builder.cpp6
-rw-r--r--yql/essentials/minikql/mkql_program_builder.h5
-rw-r--r--yql/essentials/minikql/mkql_type_builder.cpp18
-rw-r--r--yql/essentials/minikql/mkql_type_builder.h25
-rw-r--r--yql/essentials/minikql/mkql_type_builder_ut.cpp4
-rw-r--r--yql/essentials/minikql/protobuf_udf/ut/type_builder_ut.cpp2
-rw-r--r--yql/essentials/minikql/protobuf_udf/ut/value_builder_ut.cpp2
-rw-r--r--yql/essentials/minikql/ya.make1
-rw-r--r--yql/essentials/providers/common/proto/udf_resolver.proto3
-rw-r--r--yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp4
-rw-r--r--yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.cpp4
-rw-r--r--yql/essentials/providers/pure/yql_pure_provider.cpp4
-rw-r--r--yql/essentials/public/udf/udf_type_builder.h15
-rw-r--r--yql/essentials/public/udf/udf_version.h2
-rw-r--r--yql/essentials/tests/common/udf_test/test.py10
-rw-r--r--yql/essentials/tools/udf_resolver/discover.cpp5
-rw-r--r--yql/essentials/tools/udf_resolver/discover.h6
-rw-r--r--yql/essentials/tools/udf_resolver/udf_resolver.cpp2
-rw-r--r--yql/essentials/tools/udf_resolver/ya.make1
-rw-r--r--yql/essentials/udfs/common/python/bindings/py_test_engine.h2
-rw-r--r--yql/essentials/udfs/common/string/string_udf.cpp27
-rw-r--r--yql/essentials/udfs/common/string/test/canondata/result.json5
-rw-r--r--yql/essentials/udfs/common/string/test/canondata/test.test_Reverse_2025.02_/extracted11
-rw-r--r--yql/essentials/udfs/common/string/test/cases/Reverse_2025.02.cfg2
-rw-r--r--yql/essentials/udfs/common/string/test/cases/Reverse_2025.02.sql2
-rw-r--r--yql/essentials/udfs/common/string/ya.make3
-rw-r--r--yt/yql/providers/yt/gateway/file/yql_yt_file.cpp20
-rw-r--r--yt/yql/providers/yt/gateway/native/yql_yt_lambda_builder.cpp8
-rw-r--r--yt/yql/providers/yt/gateway/native/yql_yt_lambda_builder.h6
-rw-r--r--yt/yql/providers/yt/gateway/native/yql_yt_native.cpp37
-rw-r--r--yt/yql/providers/yt/job/ya.make1
-rw-r--r--yt/yql/providers/yt/job/yql_job_base.cpp6
-rw-r--r--yt/yql/providers/yt/job/yql_job_base.h6
-rw-r--r--yt/yql/providers/yt/job/yql_job_user.cpp2
-rw-r--r--yt/yql/providers/yt/lib/lambda_builder/lambda_builder.cpp12
-rw-r--r--yt/yql/providers/yt/lib/lambda_builder/lambda_builder.h5
-rw-r--r--yt/yql/providers/yt/lib/lambda_builder/ya.make1
-rw-r--r--yt/yql/providers/yt/provider/ya.make1
-rw-r--r--yt/yql/providers/yt/provider/yql_yt_datasink_exec.cpp1
-rw-r--r--yt/yql/providers/yt/provider/yql_yt_datasource_exec.cpp1
-rw-r--r--yt/yql/providers/yt/provider/yql_yt_gateway.h5
-rw-r--r--yt/yql/providers/yt/provider/yql_yt_helpers.cpp1
-rw-r--r--yt/yql/providers/yt/provider/yql_yt_io_discovery.cpp1
56 files changed, 344 insertions, 141 deletions
diff --git a/yql/essentials/core/type_ann/type_ann_core.cpp b/yql/essentials/core/type_ann/type_ann_core.cpp
index 91924ba853f..af3d5332696 100644
--- a/yql/essentials/core/type_ann/type_ann_core.cpp
+++ b/yql/essentials/core/type_ann/type_ann_core.cpp
@@ -22,6 +22,7 @@
#include <yql/essentials/minikql/dom/yson.h>
#include <yql/essentials/utils/log/log.h>
#include <yql/essentials/public/udf/udf_data_type.h>
+#include <yql/essentials/public/langver/yql_langver.h>
#include <yql/essentials/providers/common/schema/expr/yql_expr_schema.h>
#include <yql/essentials/utils/utf8.h>
@@ -7606,7 +7607,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
if (!EnsureTupleSize(*child, 1, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- } else if (settingName == "cpu" || settingName == "extraMem") {
+ } else if (settingName == "cpu" || settingName == "extraMem" ||
+ settingName == "minLang" || settingName == "maxLang") {
if (!EnsureTupleSize(*child, 2, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -7614,6 +7616,30 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
if (!EnsureAtom(child->Tail(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
+
+ if (ctx.Types.LangVer != UnknownLangVersion) {
+ if (settingName == "minLang" && ctx.Types.LangVer < FromString<NYql::TLangVersion>(child->Tail().Content())) {
+ TLangVersionBuffer buffer;
+ TStringBuf str;
+ if (!FormatLangVersion(FromString<NYql::TLangVersion>(child->Tail().Content()), buffer, str)) {
+ str = "unknown";
+ }
+
+ ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "UDF '" << name << "' is not available before version " << str));
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (settingName == "maxLang" && ctx.Types.LangVer > FromString<NYql::TLangVersion>(child->Tail().Content())) {
+ TLangVersionBuffer buffer;
+ TStringBuf str;
+ if (!FormatLangVersion(FromString<NYql::TLangVersion>(child->Tail().Content()), buffer, str)) {
+ str = "unknown";
+ }
+
+ ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "UDF '" << name << "' is not available after version " << str));
+ return IGraphTransformer::TStatus::Error;
+ }
+ }
} else {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Head().Pos()), TStringBuilder() << "Unknown setting: " << settingName));
return IGraphTransformer::TStatus::Error;
@@ -7630,6 +7656,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
description.Name = TString(name);
description.UserType = userType;
description.TypeConfig = typeConfig;
+ description.LangVer = ctx.Types.LangVer;
ctx.Types.Credentials->ForEach([&description](const TString& name, const TCredential& cred) {
description.SecureParams[TString("token:") + name] = cred.Content;
if (name.StartsWith("default_")) {
@@ -7679,6 +7706,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
cached.NormalizedUserType = description.NormalizedUserType ? description.NormalizedUserType : ctx.Expr.MakeType<TVoidExprType>();
cached.SupportsBlocks = description.SupportsBlocks;
cached.IsStrict = description.IsStrict;
+ cached.MinLangVer = description.MinLangVer;
+ cached.MaxLangVer = description.MaxLangVer;
if (name != cached.NormalizedName) {
ctx.Types.UdfTypeCache[std::make_tuple(cached.NormalizedName, TString(typeConfig), userType)] = cached;
@@ -7733,6 +7762,20 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
.Seal();
}
+ if (cached.MinLangVer != UnknownLangVersion) {
+ parent.List(settingIndex++)
+ .Atom(0, "minLang")
+ .Atom(1, cached.MinLangVer)
+ .Seal();
+ }
+
+ if (cached.MaxLangVer != UnknownLangVersion) {
+ parent.List(settingIndex++)
+ .Atom(0, "maxLang")
+ .Atom(1, cached.MaxLangVer)
+ .Seal();
+ }
+
if (settings) {
if (auto setting = GetSetting(*settings, "cpu")) {
parent.Add(settingIndex++, setting);
diff --git a/yql/essentials/core/type_ann/ya.make b/yql/essentials/core/type_ann/ya.make
index 80b44ce2799..a71db2a9c8e 100644
--- a/yql/essentials/core/type_ann/ya.make
+++ b/yql/essentials/core/type_ann/ya.make
@@ -36,6 +36,7 @@ PEERDIR(
yql/essentials/parser/pg_catalog
yql/essentials/core/sql_types
yql/essentials/parser/pg_wrapper/interface
+ yql/essentials/public/langver
library/cpp/yson/node
)
diff --git a/yql/essentials/core/yql_type_annotation.h b/yql/essentials/core/yql_type_annotation.h
index a43650a01f4..ba5d04fc170 100644
--- a/yql/essentials/core/yql_type_annotation.h
+++ b/yql/essentials/core/yql_type_annotation.h
@@ -339,6 +339,8 @@ struct TUdfCachedInfo {
const TTypeAnnotationNode* NormalizedUserType = nullptr;
bool SupportsBlocks = false;
bool IsStrict = false;
+ TLangVersion MinLangVer = UnknownLangVersion;
+ TLangVersion MaxLangVer = UnknownLangVersion;
};
const TString TypeAnnotationContextComponent = "TypeAnnotationContext";
diff --git a/yql/essentials/core/yql_udf_resolver.h b/yql/essentials/core/yql_udf_resolver.h
index dff7e5bc07a..915a7a40407 100644
--- a/yql/essentials/core/yql_udf_resolver.h
+++ b/yql/essentials/core/yql_udf_resolver.h
@@ -4,6 +4,7 @@
#include <yql/essentials/providers/common/proto/udf_resolver.pb.h>
#include <yql/essentials/core/yql_holding_file_storage.h>
+#include <yql/essentials/public/langver/yql_langver.h>
#include <yql/essentials/public/issue/yql_issue.h>
#include <yql/essentials/public/udf/udf_log.h>
@@ -42,6 +43,7 @@ public:
TString TypeConfig;
const TTypeAnnotationNode* UserType = nullptr;
THashMap<TString, TString> SecureParams;
+ NYql::TLangVersion LangVer = NYql::UnknownLangVersion;
// output
TString NormalizedName;
@@ -50,6 +52,8 @@ public:
const TTypeAnnotationNode* CallableType = nullptr;
bool SupportsBlocks = false;
bool IsStrict = false;
+ NYql::TLangVersion MinLangVer = NYql::UnknownLangVersion;
+ NYql::TLangVersion MaxLangVer = NYql::UnknownLangVersion;
TVector<TString> Messages;
};
diff --git a/yql/essentials/minikql/comp_nodes/mkql_scalar_apply.cpp b/yql/essentials/minikql/comp_nodes/mkql_scalar_apply.cpp
index 995fd76da74..28059ca4c3e 100644
--- a/yql/essentials/minikql/comp_nodes/mkql_scalar_apply.cpp
+++ b/yql/essentials/minikql/comp_nodes/mkql_scalar_apply.cpp
@@ -55,7 +55,7 @@ public:
, TimeProvider(CreateDefaultTimeProvider())
, Ctx(HolderFactory, &ValueBuilder, TComputationOptsFull(
nullptr, Alloc.Ref(), TypeEnv, *RandomProvider, *TimeProvider, NUdf::EValidatePolicy::Exception, originalContext.SecureParamsProvider,
- originalContext.CountersProvider, originalContext.LogProvider),
+ originalContext.CountersProvider, originalContext.LogProvider, originalContext.LangVer),
originalContext.Mutables, *NYql::NUdf::GetYqlMemoryPool())
{
Alloc.Ref().EnableArrowTracking = false;
diff --git a/yql/essentials/minikql/comp_nodes/mkql_udf.cpp b/yql/essentials/minikql/comp_nodes/mkql_udf.cpp
index c74c282d272..579f4ab09c0 100644
--- a/yql/essentials/minikql/comp_nodes/mkql_udf.cpp
+++ b/yql/essentials/minikql/comp_nodes/mkql_udf.cpp
@@ -50,7 +50,7 @@ public:
ui32 flags = 0;
TFunctionTypeInfo funcInfo;
const auto status = ctx.HolderFactory.GetFunctionRegistry()->FindFunctionTypeInfo(
- ctx.TypeEnv, ctx.TypeInfoHelper, ctx.CountersProvider, FunctionName, UserType->IsVoid() ? nullptr : UserType,
+ ctx.LangVer, ctx.TypeEnv, ctx.TypeInfoHelper, ctx.CountersProvider, FunctionName, UserType->IsVoid() ? nullptr : UserType,
TypeConfig, flags, Pos, ctx.SecureParamsProvider, ctx.LogProvider, &funcInfo);
if (!status.IsOk()) {
@@ -204,7 +204,7 @@ private:
ui32 flags = 0;
TFunctionTypeInfo funcInfo;
const auto status = ctx.HolderFactory.GetFunctionRegistry()->FindFunctionTypeInfo(
- ctx.TypeEnv, ctx.TypeInfoHelper, ctx.CountersProvider, FunctionName, UserType->IsVoid() ? nullptr : UserType,
+ ctx.LangVer, ctx.TypeEnv, ctx.TypeInfoHelper, ctx.CountersProvider, FunctionName, UserType->IsVoid() ? nullptr : UserType,
TypeConfig, flags, Pos, ctx.SecureParamsProvider, ctx.LogProvider, &funcInfo);
if (!status.IsOk()) {
@@ -290,7 +290,7 @@ IComputationNode* WrapUdf(TCallable& callable, const TComputationNodeFactoryCont
const auto userType = static_cast<TType*>(userTypeNode.GetNode());
const auto status = ctx.FunctionRegistry.FindFunctionTypeInfo(
- ctx.Env, ctx.TypeInfoHelper, ctx.CountersProvider, funcName, userType->IsVoid() ? nullptr : userType,
+ ctx.LangVer, ctx.Env, ctx.TypeInfoHelper, ctx.CountersProvider, funcName, userType->IsVoid() ? nullptr : userType,
typeConfig, flags, pos, ctx.SecureParamsProvider, ctx.LogProvider, &funcInfo);
if (!status.IsOk()) {
@@ -352,7 +352,7 @@ IComputationNode* WrapScriptUdf(TCallable& callable, const TComputationNodeFacto
ui32 flags = 0;
TFunctionTypeInfo funcInfo;
const auto status = ctx.FunctionRegistry.FindFunctionTypeInfo(
- ctx.Env, ctx.TypeInfoHelper, ctx.CountersProvider, funcName, userType,
+ ctx.LangVer, ctx.Env, ctx.TypeInfoHelper, ctx.CountersProvider, funcName, userType,
typeConfig, flags, pos, ctx.SecureParamsProvider, ctx.LogProvider, &funcInfo);
if (!status.IsOk()) {
diff --git a/yql/essentials/minikql/computation/mkql_computation_node.h b/yql/essentials/minikql/computation/mkql_computation_node.h
index 7238f1ed418..b1004629394 100644
--- a/yql/essentials/minikql/computation/mkql_computation_node.h
+++ b/yql/essentials/minikql/computation/mkql_computation_node.h
@@ -11,6 +11,7 @@
#include <yql/essentials/minikql/mkql_stats_registry.h>
#include <yql/essentials/minikql/mkql_terminator.h>
+#include <yql/essentials/public/langver/yql_langver.h>
#include <yql/essentials/public/udf/udf_value.h>
#include <yql/essentials/public/udf/udf_validate.h>
#include <yql/essentials/public/udf/udf_value_builder.h>
@@ -46,7 +47,7 @@ struct TComputationOpts {
struct TComputationOptsFull: public TComputationOpts {
TComputationOptsFull(IStatsRegistry* stats, TAllocState& allocState, const TTypeEnvironment& typeEnv, IRandomProvider& randomProvider,
ITimeProvider& timeProvider, NUdf::EValidatePolicy validatePolicy, const NUdf::ISecureParamsProvider* secureParamsProvider,
- NUdf::ICountersProvider* countersProvider, const NUdf::ILogProvider* logProvider)
+ NUdf::ICountersProvider* countersProvider, const NUdf::ILogProvider* logProvider, NYql::TLangVersion langver)
: TComputationOpts(stats)
, AllocState(allocState)
, TypeEnv(typeEnv)
@@ -56,6 +57,7 @@ struct TComputationOptsFull: public TComputationOpts {
, SecureParamsProvider(secureParamsProvider)
, CountersProvider(countersProvider)
, LogProvider(logProvider)
+ , LangVer(langver)
{}
TAllocState& AllocState;
@@ -66,6 +68,7 @@ struct TComputationOptsFull: public TComputationOpts {
const NUdf::ISecureParamsProvider *const SecureParamsProvider;
NUdf::ICountersProvider *const CountersProvider;
const NUdf::ILogProvider* const LogProvider;
+ const NYql::TLangVersion LangVer;
};
struct TWideFieldsInitInfo {
@@ -121,6 +124,7 @@ struct TComputationContext : public TComputationContextLLVM {
NUdf::ICountersProvider *const CountersProvider;
const NUdf::ISecureParamsProvider *const SecureParamsProvider;
const NUdf::ILogProvider* LogProvider;
+ NYql::TLangVersion LangVer = NYql::UnknownLangVersion;
TComputationContext(const THolderFactory& holderFactory,
const NUdf::IValueBuilder* builder,
@@ -280,6 +284,7 @@ struct TComputationNodeFactoryContext {
NUdf::ICountersProvider* CountersProvider;
const NUdf::ISecureParamsProvider* SecureParamsProvider;
const NUdf::ILogProvider* LogProvider;
+ NYql::TLangVersion LangVer;
const TNodeFactory& NodeFactory;
const THolderFactory& HolderFactory;
const NUdf::IValueBuilder *const Builder;
@@ -291,23 +296,24 @@ struct TComputationNodeFactoryContext {
const TNodePushBack NodePushBack;
TComputationNodeFactoryContext(
- const TNodeLocator& nodeLocator,
- const IFunctionRegistry& functionRegistry,
- const TTypeEnvironment& env,
- NUdf::ITypeInfoHelper::TPtr typeInfoHelper,
- NUdf::ICountersProvider* countersProvider,
- const NUdf::ISecureParamsProvider* secureParamsProvider,
- const NUdf::ILogProvider* logProvider,
- const TNodeFactory& nodeFactory,
- const THolderFactory& holderFactory,
- const NUdf::IValueBuilder* builder,
- NUdf::EValidateMode validateMode,
- NUdf::EValidatePolicy validatePolicy,
- EGraphPerProcess graphPerProcess,
- TComputationMutables& mutables,
- TComputationNodeOnNodeMap& elementsCache,
- TNodePushBack&& nodePushBack
- )
+ const TNodeLocator& nodeLocator,
+ const IFunctionRegistry& functionRegistry,
+ const TTypeEnvironment& env,
+ NUdf::ITypeInfoHelper::TPtr typeInfoHelper,
+ NUdf::ICountersProvider* countersProvider,
+ const NUdf::ISecureParamsProvider* secureParamsProvider,
+ const NUdf::ILogProvider* logProvider,
+ NYql::TLangVersion langver,
+ const TNodeFactory& nodeFactory,
+ const THolderFactory& holderFactory,
+ const NUdf::IValueBuilder* builder,
+ NUdf::EValidateMode validateMode,
+ NUdf::EValidatePolicy validatePolicy,
+ EGraphPerProcess graphPerProcess,
+ TComputationMutables& mutables,
+ TComputationNodeOnNodeMap& elementsCache,
+ TNodePushBack&& nodePushBack
+ )
: NodeLocator(nodeLocator)
, FunctionRegistry(functionRegistry)
, Env(env)
@@ -315,6 +321,7 @@ struct TComputationNodeFactoryContext {
, CountersProvider(countersProvider)
, SecureParamsProvider(secureParamsProvider)
, LogProvider(logProvider)
+ , LangVer(langver)
, NodeFactory(nodeFactory)
, HolderFactory(holderFactory)
, Builder(builder)
@@ -350,7 +357,8 @@ struct TComputationPatternOpts {
IStatsRegistry* stats = nullptr,
NUdf::ICountersProvider* countersProvider = nullptr,
const NUdf::ISecureParamsProvider* secureParamsProvider = nullptr,
- const NUdf::ILogProvider* logProvider = nullptr)
+ const NUdf::ILogProvider* logProvider = nullptr,
+ NYql::TLangVersion langver = NYql::UnknownLangVersion)
: AllocState(allocState)
, Env(env)
, Factory(factory)
@@ -363,6 +371,7 @@ struct TComputationPatternOpts {
, CountersProvider(countersProvider)
, SecureParamsProvider(secureParamsProvider)
, LogProvider(logProvider)
+ , LangVer(langver)
{}
void SetOptions(TComputationNodeFactory factory, const IFunctionRegistry* functionRegistry,
@@ -370,7 +379,7 @@ struct TComputationPatternOpts {
const TString& optLLVM, EGraphPerProcess graphPerProcess, IStatsRegistry* stats = nullptr,
NUdf::ICountersProvider* counters = nullptr,
const NUdf::ISecureParamsProvider* secureParamsProvider = nullptr,
- const NUdf::ILogProvider* logProvider = nullptr) {
+ const NUdf::ILogProvider* logProvider = nullptr, NYql::TLangVersion langver = NYql::UnknownLangVersion) {
Factory = factory;
FunctionRegistry = functionRegistry;
ValidateMode = validateMode;
@@ -381,6 +390,7 @@ struct TComputationPatternOpts {
CountersProvider = counters;
SecureParamsProvider = secureParamsProvider;
LogProvider = logProvider;
+ LangVer = langver;
}
void SetPatternEnv(std::shared_ptr<TPatternCacheEntry> cacheEnv) {
@@ -401,10 +411,11 @@ struct TComputationPatternOpts {
NUdf::ICountersProvider* CountersProvider = nullptr;
const NUdf::ISecureParamsProvider* SecureParamsProvider = nullptr;
const NUdf::ILogProvider* LogProvider = nullptr;
+ NYql::TLangVersion LangVer = NYql::UnknownLangVersion;
TComputationOptsFull ToComputationOptions(IRandomProvider& randomProvider, ITimeProvider& timeProvider, TAllocState* allocStatePtr = nullptr) const {
return TComputationOptsFull(Stats, allocStatePtr ? *allocStatePtr : AllocState, Env, randomProvider, timeProvider,
- ValidatePolicy, SecureParamsProvider, CountersProvider, LogProvider);
+ ValidatePolicy, SecureParamsProvider, CountersProvider, LogProvider, LangVer);
}
};
diff --git a/yql/essentials/minikql/computation/mkql_computation_node_graph.cpp b/yql/essentials/minikql/computation/mkql_computation_node_graph.cpp
index 9f2e182acc8..c2c26a2cd79 100644
--- a/yql/essentials/minikql/computation/mkql_computation_node_graph.cpp
+++ b/yql/essentials/minikql/computation/mkql_computation_node_graph.cpp
@@ -239,6 +239,7 @@ public:
, CountersProvider(opts.CountersProvider)
, SecureParamsProvider(opts.SecureParamsProvider)
, LogProvider(opts.LogProvider)
+ , LangVer(opts.LangVer)
, Factory(opts.Factory)
, FunctionRegistry(*opts.FunctionRegistry)
, ValidateMode(opts.ValidateMode)
@@ -465,6 +466,7 @@ private:
CountersProvider,
SecureParamsProvider,
LogProvider,
+ LangVer,
*NodeFactory,
*PatternNodes->HolderFactory,
PatternNodes->ValueBuilder.Get(),
@@ -562,6 +564,7 @@ private:
NUdf::ICountersProvider* CountersProvider;
const NUdf::ISecureParamsProvider* SecureParamsProvider;
const NUdf::ILogProvider* LogProvider;
+ const NYql::TLangVersion LangVer;
const TComputationNodeFactory Factory;
const IFunctionRegistry& FunctionRegistry;
TIntrusivePtr<TMemoryUsageInfo> MemInfo;
diff --git a/yql/essentials/minikql/computation/mkql_validate_ut.cpp b/yql/essentials/minikql/computation/mkql_validate_ut.cpp
index 3a1d0588ff6..b4e4d50670a 100644
--- a/yql/essentials/minikql/computation/mkql_validate_ut.cpp
+++ b/yql/essentials/minikql/computation/mkql_validate_ut.cpp
@@ -683,7 +683,8 @@ Y_UNIT_TEST_SUITE(TMiniKQLValidateTest) {
TFunctionTypeInfo funcInfo;
TType* userType = nullptr;
TStringBuf typeConfig;
- TStatus status = functionRegistry->FindFunctionTypeInfo(env, typeInfoHelper, nullptr, udfFuncName, userType, typeConfig, flags, {}, nullptr, nullptr, &funcInfo);
+ TStatus status = functionRegistry->FindFunctionTypeInfo(NYql::UnknownLangVersion, env, typeInfoHelper,
+ nullptr, udfFuncName, userType, typeConfig, flags, {}, nullptr, nullptr, &funcInfo);
MKQL_ENSURE(status.IsOk(), status.GetError());
auto type = funcInfo.FunctionType->GetReturnType();
fullValidateFunc(value, builder, type);
@@ -733,7 +734,8 @@ Y_UNIT_TEST_SUITE(TMiniKQLValidateTest) {
TType* userType = nullptr;
TStringBuf typeConfig;
NUdf::ITypeInfoHelper::TPtr typeInfoHelper(new TTypeInfoHelper);
- TStatus status = functionRegistry.FindFunctionTypeInfo(pgmBuilder.GetTypeEnvironment(), typeInfoHelper, nullptr,
+ TStatus status = functionRegistry.FindFunctionTypeInfo(NYql::UnknownLangVersion,
+ pgmBuilder.GetTypeEnvironment(), typeInfoHelper, nullptr,
udfFuncName, userType, typeConfig, flags, {}, nullptr, nullptr, &funcInfo);
MKQL_ENSURE(status.IsOk(), status.GetError());
auto callable = pgmBuilder.Udf(udfFuncName);
diff --git a/yql/essentials/minikql/computation/mkql_value_builder_ut.cpp b/yql/essentials/minikql/computation/mkql_value_builder_ut.cpp
index 4d1e90a950e..127c55bc9b2 100644
--- a/yql/essentials/minikql/computation/mkql_value_builder_ut.cpp
+++ b/yql/essentials/minikql/computation/mkql_value_builder_ut.cpp
@@ -44,7 +44,7 @@ public:
, HolderFactory(Alloc.Ref(), MemInfo, FunctionRegistry.Get())
, Builder(HolderFactory, NUdf::EValidatePolicy::Exception)
, TypeInfoHelper(new TTypeInfoHelper())
- , FunctionTypeInfoBuilder(Env, TypeInfoHelper, "", nullptr, {})
+ , FunctionTypeInfoBuilder(NYql::UnknownLangVersion, Env, TypeInfoHelper, "", nullptr, {})
{
BoolOid = NYql::NPg::LookupType("bool").TypeId;
}
diff --git a/yql/essentials/minikql/computation/ya.make.inc b/yql/essentials/minikql/computation/ya.make.inc
index 14f51bfceed..5f7cdd929bd 100644
--- a/yql/essentials/minikql/computation/ya.make.inc
+++ b/yql/essentials/minikql/computation/ya.make.inc
@@ -22,6 +22,7 @@ PEERDIR(
yql/essentials/minikql/arrow
yql/essentials/minikql/computation
yql/essentials/parser/pg_wrapper/interface
+ yql/essentials/public/langver
yql/essentials/public/udf
yql/essentials/public/udf/arrow
yql/essentials/utils
diff --git a/yql/essentials/minikql/mkql_function_registry.cpp b/yql/essentials/minikql/mkql_function_registry.cpp
index 27933e682d8..355b5a4e7e8 100644
--- a/yql/essentials/minikql/mkql_function_registry.cpp
+++ b/yql/essentials/minikql/mkql_function_registry.cpp
@@ -239,23 +239,24 @@ public:
}
TStatus FindFunctionTypeInfo(
- const TTypeEnvironment& env,
- NUdf::ITypeInfoHelper::TPtr typeInfoHelper,
- NUdf::ICountersProvider* countersProvider,
- const TStringBuf& name,
- TType* userType,
- const TStringBuf& typeConfig,
- ui32 flags,
- const NUdf::TSourcePosition& pos,
- const NUdf::ISecureParamsProvider* secureParamsProvider,
- const NUdf::ILogProvider* logProvider,
- TFunctionTypeInfo* funcInfo) const override
+ NYql::TLangVersion langver,
+ const TTypeEnvironment& env,
+ NUdf::ITypeInfoHelper::TPtr typeInfoHelper,
+ NUdf::ICountersProvider* countersProvider,
+ const TStringBuf& name,
+ TType* userType,
+ const TStringBuf& typeConfig,
+ ui32 flags,
+ const NUdf::TSourcePosition& pos,
+ const NUdf::ISecureParamsProvider* secureParamsProvider,
+ const NUdf::ILogProvider* logProvider,
+ TFunctionTypeInfo* funcInfo) const override
{
TStringBuf moduleName, funcName;
if (name.TrySplit(MODULE_NAME_DELIMITER, moduleName, funcName)) {
auto it = UdfModules_.find(moduleName);
if (it != UdfModules_.end()) {
- TFunctionTypeInfoBuilder typeInfoBuilder(env, typeInfoHelper, moduleName,
+ TFunctionTypeInfoBuilder typeInfoBuilder(langver, env, typeInfoHelper, moduleName,
(flags & NUdf::IUdfModule::TFlags::TypesOnly) ? nullptr : countersProvider, pos,
secureParamsProvider, logProvider);
const auto& module = *it->second.Impl;
@@ -412,18 +413,20 @@ public:
}
TStatus FindFunctionTypeInfo(
- const TTypeEnvironment& env,
- NUdf::ITypeInfoHelper::TPtr typeInfoHelper,
- NUdf::ICountersProvider* countersProvider,
- const TStringBuf& name,
- TType* userType,
- const TStringBuf& typeConfig,
- ui32 flags,
- const NUdf::TSourcePosition& pos,
- const NUdf::ISecureParamsProvider* secureParamsProvider,
- const NUdf::ILogProvider* logProvider,
- TFunctionTypeInfo* funcInfo) const override
+ NYql::TLangVersion langver,
+ const TTypeEnvironment& env,
+ NUdf::ITypeInfoHelper::TPtr typeInfoHelper,
+ NUdf::ICountersProvider* countersProvider,
+ const TStringBuf& name,
+ TType* userType,
+ const TStringBuf& typeConfig,
+ ui32 flags,
+ const NUdf::TSourcePosition& pos,
+ const NUdf::ISecureParamsProvider* secureParamsProvider,
+ const NUdf::ILogProvider* logProvider,
+ TFunctionTypeInfo* funcInfo) const override
{
+ Y_UNUSED(langver);
Y_UNUSED(env);
Y_UNUSED(typeInfoHelper);
Y_UNUSED(countersProvider);
diff --git a/yql/essentials/minikql/mkql_function_registry.h b/yql/essentials/minikql/mkql_function_registry.h
index bdf2e663d5f..5b648a26e8c 100644
--- a/yql/essentials/minikql/mkql_function_registry.h
+++ b/yql/essentials/minikql/mkql_function_registry.h
@@ -2,6 +2,7 @@
#include "mkql_function_metadata.h"
+#include <yql/essentials/public/langver/yql_langver.h>
#include <yql/essentials/public/udf/udf_counter.h>
#include <yql/essentials/public/udf/udf_registrator.h>
#include <yql/essentials/public/udf/udf_type_builder.h>
@@ -60,17 +61,18 @@ public:
virtual void AllowUdfPatch() = 0;
virtual TStatus FindFunctionTypeInfo(
- const TTypeEnvironment& env,
- NUdf::ITypeInfoHelper::TPtr typeInfoHelper,
- NUdf::ICountersProvider* countersProvider,
- const TStringBuf& name,
- TType* userType,
- const TStringBuf& typeConfig,
- ui32 flags,
- const NUdf::TSourcePosition& pos,
- const NUdf::ISecureParamsProvider* secureParamsProvider,
- const NUdf::ILogProvider* logProvider,
- TFunctionTypeInfo* funcInfo) const = 0;
+ NYql::TLangVersion langver,
+ const TTypeEnvironment& env,
+ NUdf::ITypeInfoHelper::TPtr typeInfoHelper,
+ NUdf::ICountersProvider* countersProvider,
+ const TStringBuf& name,
+ TType* userType,
+ const TStringBuf& typeConfig,
+ ui32 flags,
+ const NUdf::TSourcePosition& pos,
+ const NUdf::ISecureParamsProvider* secureParamsProvider,
+ const NUdf::ILogProvider* logProvider,
+ TFunctionTypeInfo* funcInfo) const = 0;
virtual TMaybe<TString> FindUdfPath(const TStringBuf& moduleName) const = 0;
diff --git a/yql/essentials/minikql/mkql_program_builder.cpp b/yql/essentials/minikql/mkql_program_builder.cpp
index 97d681db74e..8bed557c701 100644
--- a/yql/essentials/minikql/mkql_program_builder.cpp
+++ b/yql/essentials/minikql/mkql_program_builder.cpp
@@ -338,10 +338,12 @@ std::vector<TType*> ValidateBlockFlowType(const TType* flowType, bool unwrap) {
return ValidateBlockItems(wideComponents, unwrap);
}
-TProgramBuilder::TProgramBuilder(const TTypeEnvironment& env, const IFunctionRegistry& functionRegistry, bool voidWithEffects)
+TProgramBuilder::TProgramBuilder(const TTypeEnvironment& env, const IFunctionRegistry& functionRegistry,
+ bool voidWithEffects, NYql::TLangVersion langver)
: TTypeBuilder(env)
, FunctionRegistry(functionRegistry)
, VoidWithEffects(voidWithEffects)
+ , LangVer(langver)
{}
const TTypeEnvironment& TProgramBuilder::GetTypeEnvironment() const {
@@ -4097,7 +4099,7 @@ TRuntimeNode TProgramBuilder::Udf(
TFunctionTypeInfo funcInfo;
TStatus status = FunctionRegistry.FindFunctionTypeInfo(
- Env, TypeInfoHelper, nullptr, funcName, userType, typeConfig, flags, {}, nullptr, nullptr, &funcInfo);
+ LangVer, Env, TypeInfoHelper, nullptr, funcName, userType, typeConfig, flags, {}, nullptr, nullptr, &funcInfo);
MKQL_ENSURE(status.IsOk(), status.GetError());
auto runConfigType = funcInfo.RunConfigType;
diff --git a/yql/essentials/minikql/mkql_program_builder.h b/yql/essentials/minikql/mkql_program_builder.h
index e801d97da44..67a74dd48b6 100644
--- a/yql/essentials/minikql/mkql_program_builder.h
+++ b/yql/essentials/minikql/mkql_program_builder.h
@@ -4,6 +4,7 @@
#include "mkql_node.h"
#include "mkql_node_builder.h"
#include "mkql_type_builder.h"
+#include <yql/essentials/public/langver/yql_langver.h>
#include <yql/essentials/public/udf/udf_value.h>
#include <yql/essentials/core/sql_types/match_recognize.h>
@@ -142,7 +143,8 @@ std::vector<TType*> ValidateBlockFlowType(const TType* flowType, bool unwrap = t
class TProgramBuilder : public TTypeBuilder {
public:
- TProgramBuilder(const TTypeEnvironment& env, const IFunctionRegistry& functionRegistry, bool voidWithEffects = false);
+ TProgramBuilder(const TTypeEnvironment& env, const IFunctionRegistry& functionRegistry, bool voidWithEffects = false,
+ NYql::TLangVersion langver = NYql::UnknownLangVersion);
const TTypeEnvironment& GetTypeEnvironment() const;
const IFunctionRegistry& GetFunctionRegistry() const;
@@ -868,6 +870,7 @@ private:
protected:
const IFunctionRegistry& FunctionRegistry;
const bool VoidWithEffects;
+ const NYql::TLangVersion LangVer;
NUdf::ITypeInfoHelper::TPtr TypeInfoHelper;
};
diff --git a/yql/essentials/minikql/mkql_type_builder.cpp b/yql/essentials/minikql/mkql_type_builder.cpp
index 41502b144ae..59c00d17ebc 100644
--- a/yql/essentials/minikql/mkql_type_builder.cpp
+++ b/yql/essentials/minikql/mkql_type_builder.cpp
@@ -1716,6 +1716,7 @@ void TArrowType::Export(ArrowSchema* out) const {
// TFunctionTypeInfoBuilder
//////////////////////////////////////////////////////////////////////////////
TFunctionTypeInfoBuilder::TFunctionTypeInfoBuilder(
+ NYql::TLangVersion langver,
const TTypeEnvironment& env,
NUdf::ITypeInfoHelper::TPtr typeInfoHelper,
const TStringBuf& moduleName,
@@ -1723,7 +1724,8 @@ TFunctionTypeInfoBuilder::TFunctionTypeInfoBuilder(
const NUdf::TSourcePosition& pos,
const NUdf::ISecureParamsProvider* secureParamsProvider,
const NUdf::ILogProvider* logProvider)
- : Env_(env)
+ : LangVer_(langver)
+ , Env_(env)
, ReturnType_(nullptr)
, RunConfigType_(Env_.GetTypeOfVoidLazy())
, UserType_(Env_.GetTypeOfVoidLazy())
@@ -1830,6 +1832,18 @@ NUdf::TLoggerPtr TFunctionTypeInfoBuilder::MakeLogger(bool synchronized) const {
return ret;
}
+void TFunctionTypeInfoBuilder::SetMinLangVer(ui32 langver) {
+ MinLangVer_ = langver;
+
+}
+void TFunctionTypeInfoBuilder::SetMaxLangVer(ui32 langver) {
+ MaxLangVer_ = langver;
+}
+
+ui32 TFunctionTypeInfoBuilder::GetCurrentLangVer() const {
+ return LangVer_;
+}
+
NUdf::IFunctionTypeInfoBuilder1& TFunctionTypeInfoBuilder::ReturnsImpl(
NUdf::TDataTypeId typeId)
{
@@ -1948,6 +1962,8 @@ void TFunctionTypeInfoBuilder::Build(TFunctionTypeInfo* funcInfo)
funcInfo->IRFunctionName = std::move(IRFunctionName_);
funcInfo->SupportsBlocks = SupportsBlocks_;
funcInfo->IsStrict = IsStrict_;
+ funcInfo->MinLangVer = MinLangVer_;
+ funcInfo->MaxLangVer = MaxLangVer_;
}
NUdf::TType* TFunctionTypeInfoBuilder::Primitive(NUdf::TDataTypeId typeId) const
diff --git a/yql/essentials/minikql/mkql_type_builder.h b/yql/essentials/minikql/mkql_type_builder.h
index 678db9157ca..38f2869c81e 100644
--- a/yql/essentials/minikql/mkql_type_builder.h
+++ b/yql/essentials/minikql/mkql_type_builder.h
@@ -3,8 +3,10 @@
#include "mkql_node.h"
#include <yql/essentials/core/sql_types/block.h>
+#include <yql/essentials/public/langver/yql_langver.h>
#include <yql/essentials/public/udf/udf_type_builder.h>
#include <yql/essentials/public/udf/arrow/block_type_helper.h>
+#include <yql/essentials/public/langver/yql_langver.h>
#include <yql/essentials/parser/pg_wrapper/interface/compare.h>
#include <util/generic/size_literals.h>
@@ -103,6 +105,8 @@ struct TFunctionTypeInfo
bool Deterministic = true;
bool SupportsBlocks = false;
bool IsStrict = false;
+ NYql::TLangVersion MinLangVer = NYql::UnknownLangVersion;
+ NYql::TLangVersion MaxLangVer = NYql::UnknownLangVersion;
};
//////////////////////////////////////////////////////////////////////////////
@@ -121,13 +125,14 @@ class TFunctionTypeInfoBuilder: public NUdf::IFunctionTypeInfoBuilder
{
public:
TFunctionTypeInfoBuilder(
- const TTypeEnvironment& env,
- NUdf::ITypeInfoHelper::TPtr typeInfoHelper,
- const TStringBuf& moduleName,
- NUdf::ICountersProvider* countersProvider,
- const NUdf::TSourcePosition& pos,
- const NUdf::ISecureParamsProvider* secureParamsProvider = nullptr,
- const NUdf::ILogProvider* logProvider = nullptr);
+ NYql::TLangVersion langver,
+ const TTypeEnvironment& env,
+ NUdf::ITypeInfoHelper::TPtr typeInfoHelper,
+ const TStringBuf& moduleName,
+ NUdf::ICountersProvider* countersProvider,
+ const NUdf::TSourcePosition& pos,
+ const NUdf::ISecureParamsProvider* secureParamsProvider = nullptr,
+ const NUdf::ILogProvider* logProvider = nullptr);
NUdf::IFunctionTypeInfoBuilder1& ImplementationImpl(
NUdf::TUniquePtr<NUdf::IBoxedValue> impl) override;
@@ -209,8 +214,12 @@ public:
bool GetSecureParam(NUdf::TStringRef key, NUdf::TStringRef& value) const override;
NUdf::TLoggerPtr MakeLogger(bool synchronized) const override;
+ void SetMinLangVer(ui32 langver) override;
+ void SetMaxLangVer(ui32 langver) override;
+ ui32 GetCurrentLangVer() const override;
private:
+ const NYql::TLangVersion LangVer_;
const TTypeEnvironment& Env_;
NUdf::TUniquePtr<NUdf::IBoxedValue> Implementation_;
const TType* ReturnType_;
@@ -232,6 +241,8 @@ private:
TString IRFunctionName_;
bool SupportsBlocks_ = false;
bool IsStrict_ = false;
+ ui32 MinLangVer_ = NYql::UnknownLangVersion;
+ ui32 MaxLangVer_ = NYql::UnknownLangVersion;
};
class TTypeInfoHelper : public NUdf::ITypeInfoHelper
diff --git a/yql/essentials/minikql/mkql_type_builder_ut.cpp b/yql/essentials/minikql/mkql_type_builder_ut.cpp
index 3c84c6e05d6..95d47795df7 100644
--- a/yql/essentials/minikql/mkql_type_builder_ut.cpp
+++ b/yql/essentials/minikql/mkql_type_builder_ut.cpp
@@ -16,7 +16,7 @@ public:
: Alloc(__LOCATION__)
, Env(Alloc)
, TypeInfoHelper(new TTypeInfoHelper())
- , FunctionTypeInfoBuilder(Env, TypeInfoHelper, "", nullptr, {}) {
+ , FunctionTypeInfoBuilder(NYql::UnknownLangVersion, Env, TypeInfoHelper, "", nullptr, {}) {
}
private:
@@ -369,7 +369,7 @@ Y_UNIT_TEST_SUITE(TLogProviderTest) {
[this](const NUdf::TStringRef& component, NUdf::ELogLevel level, const NUdf::TStringRef& message) {
Messages.push_back({TString(component), level, TString(message)});
}))
- , FunctionTypeInfoBuilder(Env, TypeInfoHelper, "module", nullptr, {}, nullptr, withoutLog ? nullptr : LogProvider.Get())
+ , FunctionTypeInfoBuilder(NYql::UnknownLangVersion, Env, TypeInfoHelper, "module", nullptr, {}, nullptr, withoutLog ? nullptr : LogProvider.Get())
{}
TScopedAlloc Alloc;
diff --git a/yql/essentials/minikql/protobuf_udf/ut/type_builder_ut.cpp b/yql/essentials/minikql/protobuf_udf/ut/type_builder_ut.cpp
index 4e903f43a05..4782d5aae99 100644
--- a/yql/essentials/minikql/protobuf_udf/ut/type_builder_ut.cpp
+++ b/yql/essentials/minikql/protobuf_udf/ut/type_builder_ut.cpp
@@ -31,7 +31,7 @@ struct TSetup {
, Env(Alloc)
, FunctionRegistry(CreateFunctionRegistry(IBuiltinFunctionRegistry::TPtr()))
, TypeInfoHelper(new TTypeInfoHelper())
- , FunctionTypeInfoBuilder(Env, TypeInfoHelper, "", nullptr, {})
+ , FunctionTypeInfoBuilder(UnknownLangVersion, Env, TypeInfoHelper, "", nullptr, {})
, PgmBuilder(Env, *FunctionRegistry)
{
}
diff --git a/yql/essentials/minikql/protobuf_udf/ut/value_builder_ut.cpp b/yql/essentials/minikql/protobuf_udf/ut/value_builder_ut.cpp
index a3ed5f796b8..1f79035f971 100644
--- a/yql/essentials/minikql/protobuf_udf/ut/value_builder_ut.cpp
+++ b/yql/essentials/minikql/protobuf_udf/ut/value_builder_ut.cpp
@@ -37,7 +37,7 @@ struct TSetup {
, Env(Alloc)
, FunctionRegistry(CreateFunctionRegistry(IBuiltinFunctionRegistry::TPtr()))
, TypeInfoHelper(new TTypeInfoHelper())
- , FunctionTypeInfoBuilder(Env, TypeInfoHelper, "", nullptr, {})
+ , FunctionTypeInfoBuilder(UnknownLangVersion, Env, TypeInfoHelper, "", nullptr, {})
, PgmBuilder(Env, *FunctionRegistry)
, MemInfo("Test")
, HolderFactory(Alloc.Ref(), MemInfo)
diff --git a/yql/essentials/minikql/ya.make b/yql/essentials/minikql/ya.make
index c1d03fb5875..4c09b76c66f 100644
--- a/yql/essentials/minikql/ya.make
+++ b/yql/essentials/minikql/ya.make
@@ -72,6 +72,7 @@ PEERDIR(
yql/essentials/parser/pg_catalog
yql/essentials/parser/pg_wrapper/interface
yql/essentials/public/issue
+ yql/essentials/public/langver
yql/essentials/public/udf
yql/essentials/public/udf/tz
yql/essentials/types/binary_json
diff --git a/yql/essentials/providers/common/proto/udf_resolver.proto b/yql/essentials/providers/common/proto/udf_resolver.proto
index 8693e9de4e5..b827d38dbfe 100644
--- a/yql/essentials/providers/common/proto/udf_resolver.proto
+++ b/yql/essentials/providers/common/proto/udf_resolver.proto
@@ -5,6 +5,7 @@ message TFunction {
required string Name = 1;
required string TypeConfig = 2;
optional string UserType = 3;
+ optional uint32 LangVer = 4;
};
message TArgResult {
@@ -30,6 +31,8 @@ message TFunctionResult {
optional bool SupportsBlocks = 14;
optional bool IsStrict = 15;
repeated string Messages = 16;
+ optional uint32 MinLangVer = 17;
+ optional uint32 MaxLangVer = 18;
};
message TImport {
diff --git a/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp b/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp
index 9858c6ee256..e598b2d370f 100644
--- a/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp
+++ b/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp
@@ -218,6 +218,8 @@ public:
if (udf->UserType) {
udfRequest->SetUserType(WriteTypeToYson(udf->UserType));
}
+
+ udfRequest->SetLangVer(udf->LangVer);
}
TResolveResult response;
@@ -368,6 +370,8 @@ private:
}
udf->SupportsBlocks = udfRes.GetSupportsBlocks();
udf->IsStrict = udfRes.GetIsStrict();
+ udf->MinLangVer = udfRes.GetMinLangVer();
+ udf->MaxLangVer = udfRes.GetMaxLangVer();
for (const auto& m : udfRes.GetMessages()) {
udf->Messages.push_back(m);
}
diff --git a/yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.cpp b/yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.cpp
index 8b19fdc1d85..fc9304bd4fc 100644
--- a/yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.cpp
+++ b/yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.cpp
@@ -211,7 +211,7 @@ bool LoadFunctionsMetadata(const TVector<IUdfResolver::TFunction*>& functions,
logLevel);
TFunctionTypeInfo funcInfo;
- auto status = functionRegistry.FindFunctionTypeInfo(env, typeInfoHelper, nullptr,
+ auto status = functionRegistry.FindFunctionTypeInfo(udf.LangVer, env, typeInfoHelper, nullptr,
udf.Name, mkqlUserType, udf.TypeConfig, NUdf::IUdfModule::TFlags::TypesOnly, {}, secureParamsProvider.get(),
logProvider.Get(), &funcInfo);
if (!status.IsOk()) {
@@ -236,6 +236,8 @@ bool LoadFunctionsMetadata(const TVector<IUdfResolver::TFunction*>& functions,
udf.SupportsBlocks = funcInfo.SupportsBlocks;
udf.IsStrict = funcInfo.IsStrict;
+ udf.MinLangVer = funcInfo.MinLangVer;
+ udf.MaxLangVer = funcInfo.MaxLangVer;
} catch (const std::exception& e) {
auto issue = TIssue(udf.Pos, TStringBuilder()
<< "Internal error was found when udf metadata is loading for function: " << udf.Name
diff --git a/yql/essentials/providers/pure/yql_pure_provider.cpp b/yql/essentials/providers/pure/yql_pure_provider.cpp
index e5ddb87bc1f..1446b8aa380 100644
--- a/yql/essentials/providers/pure/yql_pure_provider.cpp
+++ b/yql/essentials/providers/pure/yql_pure_provider.cpp
@@ -116,7 +116,7 @@ public:
TScopedAlloc alloc(__LOCATION__, TAlignedPagePoolCounters(), State_->FunctionRegistry->SupportsSizedAllocators());
TTypeEnvironment env(alloc);
- TProgramBuilder pgmBuilder(env, *State_->FunctionRegistry);
+ TProgramBuilder pgmBuilder(env, *State_->FunctionRegistry, false, State_->Types->LangVer);
NCommon::TMkqlCommonCallableCompiler compiler;
NCommon::TMkqlBuildContext mkqlCtx(compiler, pgmBuilder, ctx);
@@ -145,7 +145,7 @@ public:
auto pattern = MakeComputationPattern(explorer, root, {}, patternOpts);
const TComputationOptsFull computeOpts(nullptr, alloc.Ref(), env,
*State_->Types->RandomProvider, *State_->Types->TimeProvider,
- NUdf::EValidatePolicy::Exception, nullptr, nullptr, logProvider.Get());
+ NUdf::EValidatePolicy::Exception, nullptr, nullptr, logProvider.Get(), State_->Types->LangVer);
auto graph = pattern->Clone(computeOpts);
const TBindTerminator bind(graph->GetTerminator());
graph->Prepare();
diff --git a/yql/essentials/public/udf/udf_type_builder.h b/yql/essentials/public/udf/udf_type_builder.h
index b5edff46eaf..9507b4bfe3b 100644
--- a/yql/essentials/public/udf/udf_type_builder.h
+++ b/yql/essentials/public/udf/udf_type_builder.h
@@ -647,7 +647,20 @@ public:
};
#endif
-#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 42)
+#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 43)
+class IFunctionTypeInfoBuilder18: public IFunctionTypeInfoBuilder17 {
+public:
+ virtual void SetMinLangVer(ui32 langver) = 0;
+
+ virtual void SetMaxLangVer(ui32 langver) = 0;
+
+ virtual ui32 GetCurrentLangVer() const = 0;
+};
+#endif
+
+#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 43)
+using IFunctionTypeInfoBuilderImpl = IFunctionTypeInfoBuilder18;
+#elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 42)
using IFunctionTypeInfoBuilderImpl = IFunctionTypeInfoBuilder17;
#elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 32)
using IFunctionTypeInfoBuilderImpl = IFunctionTypeInfoBuilder16;
diff --git a/yql/essentials/public/udf/udf_version.h b/yql/essentials/public/udf/udf_version.h
index 7b77f905431..78aaa7e3404 100644
--- a/yql/essentials/public/udf/udf_version.h
+++ b/yql/essentials/public/udf/udf_version.h
@@ -7,7 +7,7 @@ namespace NYql {
namespace NUdf {
#define CURRENT_UDF_ABI_VERSION_MAJOR 2
-#define CURRENT_UDF_ABI_VERSION_MINOR 42
+#define CURRENT_UDF_ABI_VERSION_MINOR 43
#define CURRENT_UDF_ABI_VERSION_PATCH 0
#ifdef USE_CURRENT_UDF_ABI_VERSION
diff --git a/yql/essentials/tests/common/udf_test/test.py b/yql/essentials/tests/common/udf_test/test.py
index 95b8cf1e18a..77c71f687e4 100644
--- a/yql/essentials/tests/common/udf_test/test.py
+++ b/yql/essentials/tests/common/udf_test/test.py
@@ -15,6 +15,7 @@ project_path = yatest.common.context.project_path
SOURCE_PATH = yql_utils.yql_source_path((project_path + '/cases').replace('\\', '/'))
DATA_PATH = yatest.common.output_path('cases')
ASTDIFF_PATH = yql_utils.yql_binary_path(os.getenv('YQL_ASTDIFF_PATH') or 'yql/essentials/tools/astdiff/astdiff')
+DEFAULT_LANG_VER = '2025.01'
def pytest_generate_tests(metafunc):
@@ -77,6 +78,9 @@ def test(case):
xfail = yql_utils.is_xfail(cfg)
if yql_utils.get_param('TARGET_PLATFORM') and xfail:
pytest.skip('xfail is not supported on non-default target platform')
+ langver = yql_utils.get_langver(cfg)
+ if langver is None:
+ langver = DEFAULT_LANG_VER
extra_env = dict(os.environ)
extra_env["YQL_UDF_RESOLVER"] = "1"
@@ -88,7 +92,11 @@ def test(case):
if "YA_TEST_RUNNER" in extra_env:
del extra_env["YA_TEST_RUNNER"]
- yqlrun_res = YQLRun(udfs_dir=udfs_dir, prov='yt', use_sql2yql=False, cfg_dir=os.getenv('YQL_CONFIG_DIR') or 'yql/essentials/cfg/udf_test').yql_exec(
+ yqlrun = YQLRun(udfs_dir=udfs_dir,
+ prov='yt', use_sql2yql=False,
+ cfg_dir=os.getenv('YQL_CONFIG_DIR') or 'yql/essentials/cfg/udf_test',
+ langver=langver)
+ yqlrun_res = yqlrun.yql_exec(
program=program,
run_sql=True,
tables=in_tables,
diff --git a/yql/essentials/tools/udf_resolver/discover.cpp b/yql/essentials/tools/udf_resolver/discover.cpp
index a290917a5a9..530e7d96e98 100644
--- a/yql/essentials/tools/udf_resolver/discover.cpp
+++ b/yql/essentials/tools/udf_resolver/discover.cpp
@@ -66,7 +66,7 @@ NYql::TResolveResult DoDiscover(const NYql::TResolve& inMsg, IMutableFunctionReg
TFunctionTypeInfo funcInfo;
if (!f.second.IsTypeAwareness) {
- auto status = functionRegistry.FindFunctionTypeInfo(env, typeInfoHelper,
+ auto status = functionRegistry.FindFunctionTypeInfo(NYql::UnknownLangVersion, env, typeInfoHelper,
nullptr, funcName, nullptr, nullptr, NUdf::IUdfModule::TFlags::TypesOnly, {}, nullptr, logProvider.Get(), &funcInfo);
if (!status.IsOk()) {
@@ -103,7 +103,8 @@ void Print(const NYql::TResolveResult& result, IOutputStream& out, bool printAsP
out << "UDF count: " << result.UdfsSize() << Endl;
}
-void DiscoverInFiles(const TVector<TString>& udfPaths, IOutputStream& out, bool printAsProto, NYql::NUdf::ELogLevel logLevel) {
+void DiscoverInFiles(const TVector<TString>& udfPaths, IOutputStream& out, bool printAsProto,
+ NYql::NUdf::ELogLevel logLevel) {
NYql::TResolve inMsg;
inMsg.SetRuntimeLogLevel(static_cast<ui32>(logLevel));
for (auto& path : udfPaths) {
diff --git a/yql/essentials/tools/udf_resolver/discover.h b/yql/essentials/tools/udf_resolver/discover.h
index 7c995b7f656..d32d3fa5a5a 100644
--- a/yql/essentials/tools/udf_resolver/discover.h
+++ b/yql/essentials/tools/udf_resolver/discover.h
@@ -9,8 +9,10 @@
#include <util/stream/output.h>
namespace NUdfResolver {
-void DiscoverInDir(const TString& dir, IOutputStream& out, bool printAsProto, NYql::NUdf::ELogLevel logLevel);
-void DiscoverInFile(const TString& filePath, IOutputStream& out, bool printAsProto, NYql::NUdf::ELogLevel logLevel);
+void DiscoverInDir(const TString& dir, IOutputStream& out, bool printAsProto,
+ NYql::NUdf::ELogLevel logLevel);
+void DiscoverInFile(const TString& filePath, IOutputStream& out, bool printAsProto,
+ NYql::NUdf::ELogLevel logLevel);
void Discover(IInputStream& in, IOutputStream& out, bool printAsProto);
void FillImportResultModules(const THashSet<TString>& modules, NYql::TImportResult& importRes);
;
diff --git a/yql/essentials/tools/udf_resolver/udf_resolver.cpp b/yql/essentials/tools/udf_resolver/udf_resolver.cpp
index 27ab6f87d48..03116858f7e 100644
--- a/yql/essentials/tools/udf_resolver/udf_resolver.cpp
+++ b/yql/essentials/tools/udf_resolver/udf_resolver.cpp
@@ -174,7 +174,7 @@ void ResolveUDFs() {
}
TFunctionTypeInfo funcInfo;
- auto status = newRegistry->FindFunctionTypeInfo(env, typeInfoHelper, nullptr,
+ auto status = newRegistry->FindFunctionTypeInfo(udf.GetLangVer(), env, typeInfoHelper, nullptr,
udf.GetName(), mkqlUserType, udf.GetTypeConfig(), NUdf::IUdfModule::TFlags::TypesOnly, {}, nullptr, logProvider.Get(), &funcInfo);
if (!status.IsOk()) {
udfRes->SetError(TStringBuilder() << "Failed to find UDF function: " << udf.GetName()
diff --git a/yql/essentials/tools/udf_resolver/ya.make b/yql/essentials/tools/udf_resolver/ya.make
index 05240ec3b7b..2dff7abf78f 100644
--- a/yql/essentials/tools/udf_resolver/ya.make
+++ b/yql/essentials/tools/udf_resolver/ya.make
@@ -25,6 +25,7 @@ PEERDIR(
yql/essentials/utils/backtrace
yql/essentials/utils/sys
yql/essentials/sql/pg_dummy
+ yql/essentials/public/langver
)
YQL_LAST_ABI_VERSION()
diff --git a/yql/essentials/udfs/common/python/bindings/py_test_engine.h b/yql/essentials/udfs/common/python/bindings/py_test_engine.h
index c86febf779d..ff0a61cbb8a 100644
--- a/yql/essentials/udfs/common/python/bindings/py_test_engine.h
+++ b/yql/essentials/udfs/common/python/bindings/py_test_engine.h
@@ -46,7 +46,7 @@ public:
, Alloc_(__LOCATION__)
, Env_(Alloc_)
, TypeInfoHelper_(new TTypeInfoHelper)
- , FunctionInfoBuilder_(Env_, TypeInfoHelper_, "", nullptr, {})
+ , FunctionInfoBuilder_(NYql::UnknownLangVersion, Env_, TypeInfoHelper_, "", nullptr, {})
{
HolderFactory_ = MakeHolder<THolderFactory>(
Alloc_.Ref(),
diff --git a/yql/essentials/udfs/common/string/string_udf.cpp b/yql/essentials/udfs/common/string/string_udf.cpp
index 92a1fe6b5af..9d2920ec804 100644
--- a/yql/essentials/udfs/common/string/string_udf.cpp
+++ b/yql/essentials/udfs/common/string/string_udf.cpp
@@ -1,6 +1,7 @@
#include <yql/essentials/public/udf/udf_allocator.h>
#include <yql/essentials/public/udf/udf_helpers.h>
#include <yql/essentials/public/udf/udf_value_builder.h>
+#include <yql/essentials/public/langver/yql_langver.h>
#include <library/cpp/deprecated/split/split_iterator.h>
#include <library/cpp/html/pcdata/pcdata.h>
@@ -85,18 +86,20 @@ namespace {
\
END_SIMPLE_ARROW_UDF(T##udfName, T##udfName##KernelExec::Do)
-#define STROKA_UDF(udfName, function) \
- SIMPLE_STRICT_UDF(T##udfName, TOptional<char*>(TOptional<char*>)) { \
- EMPTY_RESULT_ON_EMPTY_ARG(0) \
- const TString input(args[0].AsStringRef()); \
- try { \
- TUtf16String wide = UTF8ToWide(input); \
- function(wide); \
- return valueBuilder->NewString(WideToUTF8(wide)); \
- } catch (yexception&) { \
- return TUnboxedValue(); \
- } \
+// NOTE: The functions below are marked as deprecated, so block implementation
+// is not required for them
+SIMPLE_STRICT_UDF_OPTIONS(TReverse, TOptional<char*>(TOptional<char*>),
+ builder.SetMaxLangVer(NYql::MakeLangVersion(2025, 1))) {
+ EMPTY_RESULT_ON_EMPTY_ARG(0)
+ const TString input(args[0].AsStringRef());
+ try {
+ TUtf16String wide = UTF8ToWide(input);
+ ReverseInPlace(wide);
+ return valueBuilder->NewString(WideToUTF8(wide));
+ } catch (yexception&) {
+ return TUnboxedValue();
}
+}
#define STROKA_CASE_UDF(udfName, function) \
SIMPLE_STRICT_UDF(T##udfName, TOptional<char*>(TOptional<char*>)) { \
@@ -876,7 +879,6 @@ namespace {
STRING_UDF_MAP(STRING_UDF)
STRING_UNSAFE_UDF_MAP(STRING_UNSAFE_UDF)
- STROKA_UDF_MAP(STROKA_UDF)
STROKA_CASE_UDF_MAP(STROKA_CASE_UDF)
STROKA_ASCII_CASE_UDF_MAP(STROKA_ASCII_CASE_UDF)
STROKA_FIND_UDF_MAP(STROKA_FIND_UDF)
@@ -902,6 +904,7 @@ namespace {
STRING_STREAM_NUM_FORMATTER_UDF_MAP(STRING_REGISTER_UDF)
STRING_STREAM_TEXT_FORMATTER_UDF_MAP(STRING_REGISTER_UDF)
STRING_STREAM_HRSZ_FORMATTER_UDF_MAP(STRING_REGISTER_UDF)
+ TReverse,
TCollapseText,
TReplaceAll,
TReplaceFirst,
diff --git a/yql/essentials/udfs/common/string/test/canondata/result.json b/yql/essentials/udfs/common/string/test/canondata/result.json
index f9e3a670c2c..596f9a72180 100644
--- a/yql/essentials/udfs/common/string/test/canondata/result.json
+++ b/yql/essentials/udfs/common/string/test/canondata/result.json
@@ -89,6 +89,11 @@
"uri": "file://test.test_Replace_/results.txt"
}
],
+ "test.test[Reverse_2025.02]": [
+ {
+ "uri": "file://test.test_Reverse_2025.02_/extracted"
+ }
+ ],
"test.test[StreamFormat]": [
{
"uri": "file://test.test_StreamFormat_/results.txt"
diff --git a/yql/essentials/udfs/common/string/test/canondata/test.test_Reverse_2025.02_/extracted b/yql/essentials/udfs/common/string/test/canondata/test.test_Reverse_2025.02_/extracted
new file mode 100644
index 00000000000..2f258b4fc38
--- /dev/null
+++ b/yql/essentials/udfs/common/string/test/canondata/test.test_Reverse_2025.02_/extracted
@@ -0,0 +1,11 @@
+<tmp_path>/program.sql:<main>: Error: Type annotation
+
+ <tmp_path>/program.sql:<main>:2:1: Error: At function: RemovePrefixMembers, At function: Unordered, At function: PersistableRepr, At function: OrderedSqlProject, At function: SqlProjectItem
+ select String::Reverse('ab')
+ ^
+ <tmp_path>/program.sql:<main>:2:16: Error: At function: Apply, At function: Udf
+ select String::Reverse('ab')
+ ^
+ <tmp_path>/program.sql:<main>:2:16: Error: UDF 'String.Reverse' is not available after version 2025.01
+ select String::Reverse('ab')
+ ^ \ No newline at end of file
diff --git a/yql/essentials/udfs/common/string/test/cases/Reverse_2025.02.cfg b/yql/essentials/udfs/common/string/test/cases/Reverse_2025.02.cfg
new file mode 100644
index 00000000000..989226cf3d4
--- /dev/null
+++ b/yql/essentials/udfs/common/string/test/cases/Reverse_2025.02.cfg
@@ -0,0 +1,2 @@
+xfail
+langver 2025.02
diff --git a/yql/essentials/udfs/common/string/test/cases/Reverse_2025.02.sql b/yql/essentials/udfs/common/string/test/cases/Reverse_2025.02.sql
new file mode 100644
index 00000000000..3b388c36154
--- /dev/null
+++ b/yql/essentials/udfs/common/string/test/cases/Reverse_2025.02.sql
@@ -0,0 +1,2 @@
+select String::Reverse('ab')
+
diff --git a/yql/essentials/udfs/common/string/ya.make b/yql/essentials/udfs/common/string/ya.make
index 280686fb467..bd83c78b8c1 100644
--- a/yql/essentials/udfs/common/string/ya.make
+++ b/yql/essentials/udfs/common/string/ya.make
@@ -2,7 +2,7 @@ YQL_UDF_CONTRIB(string_udf)
YQL_ABI_VERSION(
2
- 37
+ 43
0
)
@@ -11,6 +11,7 @@ YQL_UDF_CONTRIB(string_udf)
)
PEERDIR(
+ yql/essentials/public/langver
yql/essentials/public/udf/arrow
library/cpp/deprecated/split
library/cpp/html/pcdata
diff --git a/yt/yql/providers/yt/gateway/file/yql_yt_file.cpp b/yt/yql/providers/yt/gateway/file/yql_yt_file.cpp
index a3caeb351c2..b292f281fba 100644
--- a/yt/yql/providers/yt/gateway/file/yql_yt_file.cpp
+++ b/yt/yql/providers/yt/gateway/file/yql_yt_file.cpp
@@ -164,9 +164,10 @@ struct TSession {
struct TFileYtLambdaBuilder: public TLambdaBuilder {
TFileYtLambdaBuilder(TScopedAlloc& alloc, const TSession& /*session*/,
TIntrusivePtr<IFunctionRegistry> customFunctionRegistry,
- const NUdf::ISecureParamsProvider* secureParamsProvider)
+ const NUdf::ISecureParamsProvider* secureParamsProvider,
+ TLangVersion langver)
: TLambdaBuilder(customFunctionRegistry.Get(), alloc, nullptr, CreateDeterministicRandomProvider(1), CreateDeterministicTimeProvider(10000000),
- nullptr, nullptr, secureParamsProvider)
+ nullptr, nullptr, secureParamsProvider, nullptr, langver)
, CustomFunctionRegistry_(customFunctionRegistry)
{}
@@ -445,7 +446,8 @@ public:
auto secureParamsProvider = MakeSimpleSecureParamsProvider(options.SecureParams());
TVector<TFileLinkPtr> externalFiles;
TFileYtLambdaBuilder builder(alloc, *session,
- MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(), Services_->GetFileStorage(), externalFiles), secureParamsProvider.get());
+ MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(),
+ Services_->GetFileStorage(), externalFiles), secureParamsProvider.get(), options.LangVer());
TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), *Services_->GetFunctionRegistry());
TVector<TRuntimeNode> strings;
@@ -773,7 +775,8 @@ public:
auto secureParamsProvider = MakeSimpleSecureParamsProvider(options.SecureParams());
TVector<TFileLinkPtr> externalFiles;
TFileYtLambdaBuilder builder(alloc, *session,
- MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(), Services_->GetFileStorage(), externalFiles), secureParamsProvider.get());
+ MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(),
+ Services_->GetFileStorage(), externalFiles), secureParamsProvider.get(), options.LangVer());
auto nodeFactory = GetYtFileFullFactory(Services_);
for (auto& node: nodes) {
auto data = builder.BuildLambda(*MkqlCompiler_, node, ctx);
@@ -819,7 +822,8 @@ public:
alloc.SetLimit(options.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0));
TVector<TFileLinkPtr> externalFiles;
TFileYtLambdaBuilder builder(alloc, *session,
- MakeFunctionRegistry(*Services_->GetFunctionRegistry(), {}, Services_->GetFileStorage(), externalFiles), nullptr);
+ MakeFunctionRegistry(*Services_->GetFunctionRegistry(), {}, Services_->GetFileStorage(), externalFiles),
+ nullptr, UnknownLangVersion);
TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), builder.GetFunctionRegistry());
TMkqlBuildContext ctx(*MkqlCompiler_, pgmBuilder, exprCtx);
@@ -1252,7 +1256,8 @@ private:
auto secureParamsProvider = MakeSimpleSecureParamsProvider(options.SecureParams());
TVector<TFileLinkPtr> externalFiles;
TFileYtLambdaBuilder builder(alloc, session,
- MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(), Services_->GetFileStorage(), externalFiles), secureParamsProvider.get());
+ MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(),
+ Services_->GetFileStorage(), externalFiles), secureParamsProvider.get(), options.LangVer());
auto data = builder.BuildLambda(*MkqlCompiler_, input.Ptr(), exprCtx);
auto transform = TFileTransformProvider(Services_, options.UserDataBlocks());
data = builder.TransformAndOptimizeProgram(data, transform);
@@ -1331,7 +1336,8 @@ private:
auto secureParamsProvider = MakeSimpleSecureParamsProvider(options.SecureParams());
TVector<TFileLinkPtr> externalFiles;
TFileYtLambdaBuilder builder(alloc, session,
- MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(), Services_->GetFileStorage(), externalFiles), secureParamsProvider.get());
+ MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(), Services_->GetFileStorage(),
+ externalFiles), secureParamsProvider.get(), options.LangVer());
auto data = builder.BuildLambda(*MkqlCompiler_, node, exprCtx);
auto transform = TFileTransformProvider(Services_, options.UserDataBlocks());
data = builder.TransformAndOptimizeProgram(data, transform);
diff --git a/yt/yql/providers/yt/gateway/native/yql_yt_lambda_builder.cpp b/yt/yql/providers/yt/gateway/native/yql_yt_lambda_builder.cpp
index cc9981ccf1d..9005ce4170d 100644
--- a/yt/yql/providers/yt/gateway/native/yql_yt_lambda_builder.cpp
+++ b/yt/yql/providers/yt/gateway/native/yql_yt_lambda_builder.cpp
@@ -100,14 +100,14 @@ NKikimr::NMiniKQL::TComputationNodeFactory GetGatewayNodeFactory(TCodecContext*
TNativeYtLambdaBuilder::TNativeYtLambdaBuilder(TScopedAlloc& alloc, const IFunctionRegistry* functionRegistry, const TSession& session,
- const NKikimr::NUdf::ISecureParamsProvider* secureParamsProvider)
+ const NKikimr::NUdf::ISecureParamsProvider* secureParamsProvider, TLangVersion langver)
: TLambdaBuilder(functionRegistry, alloc, nullptr,
- session.RandomProvider_, session.TimeProvider_, nullptr, nullptr, secureParamsProvider)
+ session.RandomProvider_, session.TimeProvider_, nullptr, nullptr, secureParamsProvider, nullptr, langver)
{
}
-TNativeYtLambdaBuilder::TNativeYtLambdaBuilder(TScopedAlloc& alloc, const TYtNativeServices& services, const TSession& session)
- : TNativeYtLambdaBuilder(alloc, services.FunctionRegistry, session)
+TNativeYtLambdaBuilder::TNativeYtLambdaBuilder(TScopedAlloc& alloc, const TYtNativeServices& services, const TSession& session, TLangVersion langver)
+ : TNativeYtLambdaBuilder(alloc, services.FunctionRegistry, session, nullptr, langver)
{
}
diff --git a/yt/yql/providers/yt/gateway/native/yql_yt_lambda_builder.h b/yt/yql/providers/yt/gateway/native/yql_yt_lambda_builder.h
index d147ad4c650..7985054a6e0 100644
--- a/yt/yql/providers/yt/gateway/native/yql_yt_lambda_builder.h
+++ b/yt/yql/providers/yt/gateway/native/yql_yt_lambda_builder.h
@@ -31,9 +31,11 @@ struct TSession;
struct TNativeYtLambdaBuilder: public TLambdaBuilder {
TNativeYtLambdaBuilder(NKikimr::NMiniKQL::TScopedAlloc& alloc, const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
- const TSession& session, const NKikimr::NUdf::ISecureParamsProvider* secureParamsProvider = nullptr);
+ const TSession& session, const NKikimr::NUdf::ISecureParamsProvider* secureParamsProvider,
+ TLangVersion langver);
- TNativeYtLambdaBuilder(NKikimr::NMiniKQL::TScopedAlloc& alloc, const TYtNativeServices& services, const TSession& session);
+ TNativeYtLambdaBuilder(NKikimr::NMiniKQL::TScopedAlloc& alloc, const TYtNativeServices& services, const TSession& session,
+ TLangVersion langver);
TString BuildLambdaWithIO(const NCommon::IMkqlCallableCompiler& compiler, NNodes::TCoLambda lambda, TExprContext& exprCtx);
};
diff --git a/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp b/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp
index 2465ceac4cd..460b607e01a 100644
--- a/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp
+++ b/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp
@@ -1006,7 +1006,7 @@ public:
TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(),
Services_.FunctionRegistry->SupportsSizedAllocators());
alloc.SetLimit(options.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0));
- TNativeYtLambdaBuilder builder(alloc, Services_, *session);
+ TNativeYtLambdaBuilder builder(alloc, Services_, *session, options.LangVer());
TVector<TRuntimeNode> tupleNodes;
for (auto& node: nodes) {
tupleNodes.push_back(builder.BuildLambda(*MkqlCompiler_, node, ctx));
@@ -2314,7 +2314,7 @@ private:
TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(),
execCtx->FunctionRegistry_->SupportsSizedAllocators());
alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0));
- TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_);
+ TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_, nullptr, execCtx->Options_.LangVer());
TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), *execCtx->FunctionRegistry_);
TRuntimeNode root = DeserializeRuntimeNode(filterLambda, builder.GetTypeEnvironment());
@@ -3493,7 +3493,7 @@ private:
TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(),
Services_.FunctionRegistry->SupportsSizedAllocators());
alloc.SetLimit(options.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0));
- TNativeYtLambdaBuilder builder(alloc, Services_, *session);
+ TNativeYtLambdaBuilder builder(alloc, Services_, *session, options.LangVer());
auto rootNode = builder.BuildLambda(*MkqlCompiler_, result.Input().Ptr(), ctx);
hasListResult = rootNode.GetStaticType()->IsList();
lambda = SerializeRuntimeNode(rootNode, builder.GetTypeEnvironment());
@@ -3869,7 +3869,7 @@ private:
TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(),
execCtx->FunctionRegistry_->SupportsSizedAllocators());
alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0));
- TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_);
+ TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_, nullptr, execCtx->Options_.LangVer());
TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), *execCtx->FunctionRegistry_);
TGatewayTransformer transform(execCtx, entry, pgmBuilder, *tmpFiles);
size_t nodeCount = 0;
@@ -3881,6 +3881,7 @@ private:
job->SetOptLLVM(execCtx->Options_.OptLLVM());
job->SetUdfValidateMode(execCtx->Options_.UdfValidateMode());
job->SetRuntimeLogLevel(execCtx->Options_.RuntimeLogLevel());
+ job->SetLangVer(execCtx->Options_.LangVer());
transform.ApplyJobProps(*job);
transform.ApplyUserJobSpec(userJobSpec, testRun);
@@ -3956,7 +3957,7 @@ private:
TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(),
execCtx->FunctionRegistry_->SupportsSizedAllocators());
alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0));
- TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_);
+ TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_, execCtx->Options_.LangVer());
mapLambda = builder.BuildLambdaWithIO(*MkqlCompiler_, map.Mapper(), ctx);
}
@@ -4093,7 +4094,7 @@ private:
TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(),
execCtx->FunctionRegistry_->SupportsSizedAllocators());
alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0));
- TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_);
+ TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_, nullptr, execCtx->Options_.LangVer());
TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), *execCtx->FunctionRegistry_);
TGatewayTransformer transform(execCtx, entry, pgmBuilder, *tmpFiles);
size_t nodeCount = 0;
@@ -4105,6 +4106,7 @@ private:
job->SetOptLLVM(execCtx->Options_.OptLLVM());
job->SetUdfValidateMode(execCtx->Options_.UdfValidateMode());
job->SetRuntimeLogLevel(execCtx->Options_.RuntimeLogLevel());
+ job->SetLangVer(execCtx->Options_.LangVer());
transform.ApplyJobProps(*job);
transform.ApplyUserJobSpec(userJobSpec, testRun);
FillUserJobSpec(userJobSpec, execCtx, extraUsage, transform.GetUsedMemory(), execCtx->EstimateLLVMMem(nodeCount), testRun);
@@ -4163,7 +4165,7 @@ private:
TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(),
execCtx->FunctionRegistry_->SupportsSizedAllocators());
alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0));
- TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_);
+ TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_, execCtx->Options_.LangVer());
reduceLambda = builder.BuildLambdaWithIO(*MkqlCompiler_, reduce.Reducer(), ctx);
}
@@ -4347,7 +4349,7 @@ private:
TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(),
execCtx->FunctionRegistry_->SupportsSizedAllocators());
alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0));
- TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_);
+ TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_, nullptr, execCtx->Options_.LangVer());
TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), *execCtx->FunctionRegistry_);
TGatewayTransformer transform(execCtx, entry, pgmBuilder, *tmpFiles);
size_t nodeCount = 0;
@@ -4359,6 +4361,7 @@ private:
mapJob->SetOptLLVM(execCtx->Options_.OptLLVM());
mapJob->SetUdfValidateMode(execCtx->Options_.UdfValidateMode());
mapJob->SetRuntimeLogLevel(execCtx->Options_.RuntimeLogLevel());
+ mapJob->SetLangVer(execCtx->Options_.LangVer());
transform.ApplyJobProps(*mapJob);
transform.ApplyUserJobSpec(mapUserJobSpec, testRun);
@@ -4377,7 +4380,7 @@ private:
TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(),
execCtx->FunctionRegistry_->SupportsSizedAllocators());
alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0));
- TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_);
+ TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_, nullptr, execCtx->Options_.LangVer());
TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), *execCtx->FunctionRegistry_);
TGatewayTransformer transform(execCtx, entry, pgmBuilder, *tmpFiles);
size_t nodeCount = 0;
@@ -4389,6 +4392,7 @@ private:
reduceJob->SetOptLLVM(execCtx->Options_.OptLLVM());
reduceJob->SetUdfValidateMode(execCtx->Options_.UdfValidateMode());
reduceJob->SetRuntimeLogLevel(execCtx->Options_.RuntimeLogLevel());
+ reduceJob->SetLangVer(execCtx->Options_.LangVer());
transform.ApplyJobProps(*reduceJob);
transform.ApplyUserJobSpec(reduceUserJobSpec, testRun);
FillUserJobSpec(reduceUserJobSpec, execCtx, reduceExtraUsage, transform.GetUsedMemory(), execCtx->EstimateLLVMMem(nodeCount), testRun);
@@ -4532,7 +4536,7 @@ private:
TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(),
execCtx->FunctionRegistry_->SupportsSizedAllocators());
alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0));
- TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_);
+ TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_, nullptr, execCtx->Options_.LangVer());
TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), *execCtx->FunctionRegistry_);
TGatewayTransformer transform(execCtx, entry, pgmBuilder, *tmpFiles);
size_t nodeCount = 0;
@@ -4544,6 +4548,7 @@ private:
reduceJob->SetOptLLVM(execCtx->Options_.OptLLVM());
reduceJob->SetUdfValidateMode(execCtx->Options_.UdfValidateMode());
reduceJob->SetRuntimeLogLevel(execCtx->Options_.RuntimeLogLevel());
+ reduceJob->SetLangVer(execCtx->Options_.LangVer());
transform.ApplyJobProps(*reduceJob);
transform.ApplyUserJobSpec(reduceUserJobSpec, testRun);
FillUserJobSpec(reduceUserJobSpec, execCtx, reduceExtraUsage, transform.GetUsedMemory(), execCtx->EstimateLLVMMem(nodeCount), testRun);
@@ -4669,7 +4674,7 @@ private:
TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(),
execCtx->FunctionRegistry_->SupportsSizedAllocators());
alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0));
- TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_);
+ TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_, execCtx->Options_.LangVer());
mapLambda = builder.BuildLambdaWithIO(*MkqlCompiler_, mapReduce.Mapper().Cast<TCoLambda>(), ctx);
mapInputType = NCommon::WriteTypeToYson(GetSequenceItemType(mapReduce.Input().Size() == 1U ?
TExprBase(mapReduce.Input().Item(0)) : TExprBase(mapReduce.Mapper().Cast<TCoLambda>().Args().Arg(0)), true));
@@ -4688,7 +4693,7 @@ private:
TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(),
execCtx->FunctionRegistry_->SupportsSizedAllocators());
alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0));
- TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_);
+ TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_, execCtx->Options_.LangVer());
reduceLambda = builder.BuildLambdaWithIO(*MkqlCompiler_, mapReduce.Reducer(), ctx);
}
TExpressionResorceUsage reduceExtraUsage = execCtx->ScanExtraResourceUsage(mapReduce.Reducer().Body().Ref(), false);
@@ -4831,7 +4836,7 @@ private:
TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(),
execCtx->FunctionRegistry_->SupportsSizedAllocators());
alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0));
- TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_);
+ TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_, nullptr, execCtx->Options_.LangVer());
TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), *execCtx->FunctionRegistry_);
TGatewayTransformer transform(execCtx, entry, pgmBuilder, *tmpFiles);
transform.SetTwoPhaseTransform();
@@ -4872,6 +4877,7 @@ private:
job->SetOptLLVM(execCtx->Options_.OptLLVM());
job->SetUdfValidateMode(execCtx->Options_.UdfValidateMode());
job->SetRuntimeLogLevel(execCtx->Options_.RuntimeLogLevel());
+ job->SetLangVer(execCtx->Options_.LangVer());
const auto nativeTypeCompat = execCtx->Options_.Config()->NativeYtTypeCompatibility.Get(execCtx->Cluster_).GetOrElse(NTCF_LEGACY);
job->SetOutSpec(execCtx->GetOutSpec(!useSkiff, nativeTypeCompat));
job->SetUseSkiff(useSkiff, 0);
@@ -4935,7 +4941,7 @@ private:
TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(),
Services_.FunctionRegistry->SupportsSizedAllocators());
alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0));
- TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_);
+ TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_, execCtx->Options_.LangVer());
lambda = builder.BuildLambdaWithIO(*MkqlCompiler_, fill.Content(), ctx);
}
auto extraUsage = execCtx->ScanExtraResourceUsage(fill.Content().Ref(), false);
@@ -5432,7 +5438,7 @@ private:
execCtx->FunctionRegistry_->SupportsSizedAllocators());
alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0));
auto secureParamsProvider = MakeSimpleSecureParamsProvider(execCtx->Options_.SecureParams());
- TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_, secureParamsProvider.get());
+ TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_, secureParamsProvider.get(), execCtx->Options_.LangVer());
THolder<TCodecContext> codecCtx;
TString pathPrefix;
TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), *execCtx->FunctionRegistry_);
@@ -5507,6 +5513,7 @@ private:
job->SetOptLLVM(execCtx->Options_.OptLLVM());
job->SetUdfValidateMode(execCtx->Options_.UdfValidateMode());
job->SetRuntimeLogLevel(execCtx->Options_.RuntimeLogLevel());
+ job->SetLangVer(execCtx->Options_.LangVer());
mapOpSpec.AddInput(tmpTable);
mapOpSpec.AddOutput(tmpTable);
diff --git a/yt/yql/providers/yt/job/ya.make b/yt/yql/providers/yt/job/ya.make
index fe892d4f71f..f67ac0d8c2a 100644
--- a/yt/yql/providers/yt/job/ya.make
+++ b/yt/yql/providers/yt/job/ya.make
@@ -18,6 +18,7 @@ PEERDIR(
yt/cpp/mapreduce/interface
yt/cpp/mapreduce/library/user_job_statistics
yql/essentials/minikql/comp_nodes
+ yql/essentials/public/langver
yql/essentials/public/udf
yql/essentials/utils
yql/essentials/utils/backtrace
diff --git a/yt/yql/providers/yt/job/yql_job_base.cpp b/yt/yql/providers/yt/job/yql_job_base.cpp
index 00d96a9ca2e..0874eb45244 100644
--- a/yt/yql/providers/yt/job/yql_job_base.cpp
+++ b/yt/yql/providers/yt/job/yql_job_base.cpp
@@ -286,7 +286,8 @@ void TYqlJobBase::Save(IOutputStream& s) const {
UdfValidateMode,
OptLLVM,
TableNames,
- RuntimeLogLevel
+ RuntimeLogLevel,
+ LangVer
);
}
@@ -297,7 +298,8 @@ void TYqlJobBase::Load(IInputStream& s) {
UdfValidateMode,
OptLLVM,
TableNames,
- RuntimeLogLevel
+ RuntimeLogLevel,
+ LangVer
);
}
diff --git a/yt/yql/providers/yt/job/yql_job_base.h b/yt/yql/providers/yt/job/yql_job_base.h
index e9bf6b7c66e..eb9e02587dc 100644
--- a/yt/yql/providers/yt/job/yql_job_base.h
+++ b/yt/yql/providers/yt/job/yql_job_base.h
@@ -1,6 +1,7 @@
#pragma once
#include <yql/essentials/providers/common/codec/yql_codec.h>
+#include <yql/essentials/public/langver/yql_langver.h>
#include <yql/essentials/public/udf/udf_validate.h>
#include <yql/essentials/public/udf/udf_counter.h>
#include <yql/essentials/minikql/mkql_node_visitor.h>
@@ -74,6 +75,10 @@ public:
RuntimeLogLevel = level;
}
+ void SetLangVer(TLangVersion langver) {
+ LangVer = langver;
+ }
+
void Do(const NYT::TRawJobContext& jobContext) override;
void Save(IOutputStream& stream) const override;
void Load(IInputStream& stream) override;
@@ -93,6 +98,7 @@ protected:
TString OptLLVM;
TVector<TString> TableNames;
NUdf::ELogLevel RuntimeLogLevel = NUdf::ELogLevel::Info;
+ TLangVersion LangVer = UnknownLangVersion;
// End serializable part
ui64 StartCycles = 0;
diff --git a/yt/yql/providers/yt/job/yql_job_user.cpp b/yt/yql/providers/yt/job/yql_job_user.cpp
index 0b355e6609d..edbcce66509 100644
--- a/yt/yql/providers/yt/job/yql_job_user.cpp
+++ b/yt/yql/providers/yt/job/yql_job_user.cpp
@@ -150,7 +150,7 @@ void TYqlUserJob::DoImpl(const TFile& inHandle, const TVector<TFile>& outHandles
TLambdaBuilder builder(FunctionRegistry.Get(), *Alloc,
Env.Get(), RandomProvider.Get(), TimeProvider.Get(), JobStats.Get(), &JobCountersProvider,
- SecureParamsProvider.Get(), LogProvider.Get());
+ SecureParamsProvider.Get(), LogProvider.Get(), LangVer);
TType* itemType = nullptr;
if (InputType) {
diff --git a/yt/yql/providers/yt/lib/lambda_builder/lambda_builder.cpp b/yt/yql/providers/yt/lib/lambda_builder/lambda_builder.cpp
index d706a71a7a3..cfca9dcbb8a 100644
--- a/yt/yql/providers/yt/lib/lambda_builder/lambda_builder.cpp
+++ b/yt/yql/providers/yt/lib/lambda_builder/lambda_builder.cpp
@@ -23,7 +23,8 @@ TLambdaBuilder::TLambdaBuilder(const NKikimr::NMiniKQL::IFunctionRegistry* funct
NKikimr::NMiniKQL::IStatsRegistry* jobStats,
NKikimr::NUdf::ICountersProvider* counters,
const NKikimr::NUdf::ISecureParamsProvider* secureParamsProvider,
- const NKikimr::NUdf::ILogProvider* logProvider)
+ const NKikimr::NUdf::ILogProvider* logProvider,
+ TLangVersion langVer)
: FunctionRegistry(functionRegistry)
, Alloc(alloc)
, RandomProvider(randomProvider)
@@ -32,6 +33,7 @@ TLambdaBuilder::TLambdaBuilder(const NKikimr::NMiniKQL::IFunctionRegistry* funct
, Counters(counters)
, SecureParamsProvider(secureParamsProvider)
, LogProvider(logProvider)
+ , LangVer(langVer)
, Env(env)
{
}
@@ -52,7 +54,7 @@ const NKikimr::NMiniKQL::TTypeEnvironment* TLambdaBuilder::CreateTypeEnv() const
TRuntimeNode TLambdaBuilder::BuildLambda(const IMkqlCallableCompiler& compiler, const TExprNode::TPtr& lambdaNode,
TExprContext& exprCtx, TArgumentsMap&& arguments) const
{
- TProgramBuilder pgmBuilder(GetTypeEnvironment(), *FunctionRegistry);
+ TProgramBuilder pgmBuilder(GetTypeEnvironment(), *FunctionRegistry, false, LangVer);
TMkqlBuildContext ctx(compiler, pgmBuilder, exprCtx, lambdaNode->UniqueId(), std::move(arguments));
return MkqlBuildExpr(*lambdaNode, ctx);
}
@@ -184,7 +186,7 @@ THolder<IComputationGraph> TLambdaBuilder::BuildGraph(
TComputationPatternOpts patternOpts(Alloc.Ref(), GetTypeEnvironment());
patternOpts.SetOptions(factory, FunctionRegistry, validateMode, validatePolicy,
optLLVM, graphPerProcess, JobStats, Counters,
- SecureParamsProvider, LogProvider);
+ SecureParamsProvider, LogProvider, LangVer);
auto preparePatternFunc = [&]() {
if (serialized) {
auto tupleRunTimeNodes = DeserializeRuntimeNode(serialized, GetTypeEnvironment());
@@ -206,13 +208,13 @@ THolder<IComputationGraph> TLambdaBuilder::BuildGraph(
YQL_ENSURE(pattern);
const TComputationOptsFull computeOpts(JobStats, Alloc.Ref(), GetTypeEnvironment(), *randomProvider, *timeProvider,
- validatePolicy, SecureParamsProvider, Counters, LogProvider);
+ validatePolicy, SecureParamsProvider, Counters, LogProvider, LangVer);
auto graph = pattern->Clone(computeOpts);
return MakeHolder<TComputationGraphProxy>(std::move(pattern), std::move(graph));
}
TRuntimeNode TLambdaBuilder::MakeTuple(const TVector<TRuntimeNode>& items) const {
- TProgramBuilder pgmBuilder(GetTypeEnvironment(), *FunctionRegistry);
+ TProgramBuilder pgmBuilder(GetTypeEnvironment(), *FunctionRegistry, false, LangVer);
return pgmBuilder.NewTuple(items);
}
diff --git a/yt/yql/providers/yt/lib/lambda_builder/lambda_builder.h b/yt/yql/providers/yt/lib/lambda_builder/lambda_builder.h
index c1f7aca1b41..20f3226a2f8 100644
--- a/yt/yql/providers/yt/lib/lambda_builder/lambda_builder.h
+++ b/yt/yql/providers/yt/lib/lambda_builder/lambda_builder.h
@@ -10,6 +10,7 @@
#include <yql/essentials/minikql/mkql_node.h>
#include <yql/essentials/minikql/mkql_alloc.h>
#include <yql/essentials/public/udf/udf_validate.h>
+#include <yql/essentials/public/langver/yql_langver.h>
#include <library/cpp/random_provider/random_provider.h>
#include <library/cpp/time_provider/time_provider.h>
@@ -35,7 +36,8 @@ public:
NKikimr::NMiniKQL::IStatsRegistry* jobStats = nullptr,
NKikimr::NUdf::ICountersProvider* counters = nullptr,
const NKikimr::NUdf::ISecureParamsProvider *secureParamsProvider = nullptr,
- const NKikimr::NUdf::ILogProvider* logProvider = nullptr);
+ const NKikimr::NUdf::ILogProvider* logProvider = nullptr,
+ TLangVersion langver = UnknownLangVersion);
~TLambdaBuilder();
@@ -108,6 +110,7 @@ protected:
NKikimr::NUdf::ICountersProvider* const Counters;
const NKikimr::NUdf::ISecureParamsProvider* SecureParamsProvider;
const NKikimr::NUdf::ILogProvider* LogProvider;
+ const TLangVersion LangVer;
/// TODO: remove?
void SetExternalEnv(const NKikimr::NMiniKQL::TTypeEnvironment* env);
diff --git a/yt/yql/providers/yt/lib/lambda_builder/ya.make b/yt/yql/providers/yt/lib/lambda_builder/ya.make
index 306049a0eee..a0f73ed76be 100644
--- a/yt/yql/providers/yt/lib/lambda_builder/ya.make
+++ b/yt/yql/providers/yt/lib/lambda_builder/ya.make
@@ -12,6 +12,7 @@ PEERDIR(
yql/essentials/public/udf
yql/essentials/utils
yql/essentials/providers/common/mkql
+ yql/essentials/public/langver
)
YQL_LAST_ABI_VERSION()
diff --git a/yt/yql/providers/yt/provider/ya.make b/yt/yql/providers/yt/provider/ya.make
index 74cbe7ac426..e705a3476a2 100644
--- a/yt/yql/providers/yt/provider/ya.make
+++ b/yt/yql/providers/yt/provider/ya.make
@@ -80,6 +80,7 @@ PEERDIR(
yt/cpp/mapreduce/interface
yql/essentials/ast
yql/essentials/core/extract_predicate
+ yql/essentials/public/langver
yql/essentials/public/udf
yql/essentials/public/udf/tz
yql/essentials/sql
diff --git a/yt/yql/providers/yt/provider/yql_yt_datasink_exec.cpp b/yt/yql/providers/yt/provider/yql_yt_datasink_exec.cpp
index 05e0ca5caa0..74905bb7b32 100644
--- a/yt/yql/providers/yt/provider/yql_yt_datasink_exec.cpp
+++ b/yt/yql/providers/yt/provider/yql_yt_datasink_exec.cpp
@@ -314,6 +314,7 @@ private:
.OperationHash(operationHash)
.SecureParams(secureParams)
.RuntimeLogLevel(State_->Types->RuntimeLogLevel)
+ .LangVer(State_->Types->LangVer)
.AdditionalSecurityTags(addSecTags)
);
}
diff --git a/yt/yql/providers/yt/provider/yql_yt_datasource_exec.cpp b/yt/yql/providers/yt/provider/yql_yt_datasource_exec.cpp
index c4ed2d2a34d..971b66209ed 100644
--- a/yt/yql/providers/yt/provider/yql_yt_datasource_exec.cpp
+++ b/yt/yql/providers/yt/provider/yql_yt_datasource_exec.cpp
@@ -200,6 +200,7 @@ protected:
.OperationHash(operationHash)
.SecureParams(secureParams)
.RuntimeLogLevel(State_->Types->RuntimeLogLevel)
+ .LangVer(State_->Types->LangVer)
);
return WrapFuture(future, [](const IYtGateway::TResOrPullResult& res, const TExprNode::TPtr& input, TExprContext& ctx) {
diff --git a/yt/yql/providers/yt/provider/yql_yt_gateway.h b/yt/yql/providers/yt/provider/yql_yt_gateway.h
index e1b49abf7b6..a40f1f863f9 100644
--- a/yt/yql/providers/yt/provider/yql_yt_gateway.h
+++ b/yt/yql/providers/yt/provider/yql_yt_gateway.h
@@ -13,6 +13,7 @@
#include <yql/essentials/core/yql_type_annotation.h>
#include <yql/essentials/core/yql_execution.h>
#include <yql/essentials/core/file_storage/storage.h>
+#include <yql/essentials/public/langver/yql_langver.h>
#include <yt/cpp/mapreduce/interface/common.h>
@@ -251,6 +252,7 @@ public:
OPTION_FIELD(TPosition, Pos)
OPTION_FIELD(TSecureParams, SecureParams)
OPTION_FIELD_DEFAULT(NUdf::ELogLevel, RuntimeLogLevel, NUdf::ELogLevel::Info)
+ OPTION_FIELD_DEFAULT(TLangVersion, LangVer, UnknownLangVersion)
};
struct TTableRangeResult : public NCommon::TOperationResult {
@@ -360,6 +362,7 @@ public:
OPTION_FIELD(TString, OperationHash)
OPTION_FIELD(TSecureParams, SecureParams)
OPTION_FIELD_DEFAULT(NUdf::ELogLevel, RuntimeLogLevel, NUdf::ELogLevel::Info)
+ OPTION_FIELD_DEFAULT(TLangVersion, LangVer, UnknownLangVersion)
};
struct TResOrPullResult : public NCommon::TOperationResult {
@@ -386,6 +389,7 @@ public:
OPTION_FIELD(TString, OperationHash)
OPTION_FIELD(TSecureParams, SecureParams)
OPTION_FIELD_DEFAULT(NUdf::ELogLevel, RuntimeLogLevel, NUdf::ELogLevel::Info)
+ OPTION_FIELD_DEFAULT(TLangVersion, LangVer, UnknownLangVersion)
OPTION_FIELD_DEFAULT(TSet<TString>, AdditionalSecurityTags, {})
};
@@ -432,6 +436,7 @@ public:
OPTION_FIELD(TString, OperationHash)
OPTION_FIELD(TSecureParams, SecureParams)
OPTION_FIELD_DEFAULT(NUdf::ELogLevel, RuntimeLogLevel, NUdf::ELogLevel::Info)
+ OPTION_FIELD_DEFAULT(TLangVersion, LangVer, UnknownLangVersion)
};
struct TCalcResult : public NCommon::TOperationResult {
diff --git a/yt/yql/providers/yt/provider/yql_yt_helpers.cpp b/yt/yql/providers/yt/provider/yql_yt_helpers.cpp
index 887b57d2044..4ee06a21708 100644
--- a/yt/yql/providers/yt/provider/yql_yt_helpers.cpp
+++ b/yt/yql/providers/yt/provider/yql_yt_helpers.cpp
@@ -950,6 +950,7 @@ std::pair<IGraphTransformer::TStatus, TAsyncTransformCallbackFuture> CalculateNo
.OperationHash(calcHash)
.SecureParams(secureParams)
.RuntimeLogLevel(state->Types->RuntimeLogLevel)
+ .LangVer(state->Types->LangVer)
);
return WrapFutureCallback(future, [state, calcNodes](const IYtGateway::TCalcResult& res, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
YQL_ENSURE(res.Data.size() == calcNodes.size());
diff --git a/yt/yql/providers/yt/provider/yql_yt_io_discovery.cpp b/yt/yql/providers/yt/provider/yql_yt_io_discovery.cpp
index feb5f75cb74..fd31f57950c 100644
--- a/yt/yql/providers/yt/provider/yql_yt_io_discovery.cpp
+++ b/yt/yql/providers/yt/provider/yql_yt_io_discovery.cpp
@@ -307,6 +307,7 @@ public:
.Config(State_->Configuration->Snapshot())
.OptLLVM(State_->Types->OptLLVM.GetOrElse(TString()))
.RuntimeLogLevel(State_->Types->RuntimeLogLevel)
+ .LangVer(State_->Types->LangVer)
.Pos(x.second.first)
);
allFutures.push_back(result.IgnoreResult());