diff options
author | vvvv <vvvv@yandex-team.com> | 2025-04-21 20:53:56 +0300 |
---|---|---|
committer | vvvv <vvvv@yandex-team.com> | 2025-04-21 21:26:59 +0300 |
commit | 108c1aa409626aa29ca728edafd01fbbd7a1b00e (patch) | |
tree | b88c1e3b690a29b0ed4f505f05990d656d6373ec | |
parent | 6b17ad8c52e16d0088291781e1b5a9e28d028b9b (diff) | |
download | ydb-108c1aa409626aa29ca728edafd01fbbd7a1b00e.tar.gz |
YQL-19861 abi, udf resolvers, test, pass via yt gateways
commit_hash:6e3f5fac6a8598586987b52d749644d1ce1fccbe
56 files changed, 344 insertions, 141 deletions
diff --git a/yql/essentials/core/type_ann/type_ann_core.cpp b/yql/essentials/core/type_ann/type_ann_core.cpp index 91924ba853f..af3d5332696 100644 --- a/yql/essentials/core/type_ann/type_ann_core.cpp +++ b/yql/essentials/core/type_ann/type_ann_core.cpp @@ -22,6 +22,7 @@ #include <yql/essentials/minikql/dom/yson.h> #include <yql/essentials/utils/log/log.h> #include <yql/essentials/public/udf/udf_data_type.h> +#include <yql/essentials/public/langver/yql_langver.h> #include <yql/essentials/providers/common/schema/expr/yql_expr_schema.h> #include <yql/essentials/utils/utf8.h> @@ -7606,7 +7607,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> if (!EnsureTupleSize(*child, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } - } else if (settingName == "cpu" || settingName == "extraMem") { + } else if (settingName == "cpu" || settingName == "extraMem" || + settingName == "minLang" || settingName == "maxLang") { if (!EnsureTupleSize(*child, 2, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -7614,6 +7616,30 @@ template <NKikimr::NUdf::EDataSlot DataSlot> if (!EnsureAtom(child->Tail(), ctx.Expr)) { return IGraphTransformer::TStatus::Error; } + + if (ctx.Types.LangVer != UnknownLangVersion) { + if (settingName == "minLang" && ctx.Types.LangVer < FromString<NYql::TLangVersion>(child->Tail().Content())) { + TLangVersionBuffer buffer; + TStringBuf str; + if (!FormatLangVersion(FromString<NYql::TLangVersion>(child->Tail().Content()), buffer, str)) { + str = "unknown"; + } + + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "UDF '" << name << "' is not available before version " << str)); + return IGraphTransformer::TStatus::Error; + } + + if (settingName == "maxLang" && ctx.Types.LangVer > FromString<NYql::TLangVersion>(child->Tail().Content())) { + TLangVersionBuffer buffer; + TStringBuf str; + if (!FormatLangVersion(FromString<NYql::TLangVersion>(child->Tail().Content()), buffer, str)) { + str = "unknown"; + } + + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "UDF '" << name << "' is not available after version " << str)); + return IGraphTransformer::TStatus::Error; + } + } } else { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Head().Pos()), TStringBuilder() << "Unknown setting: " << settingName)); return IGraphTransformer::TStatus::Error; @@ -7630,6 +7656,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> description.Name = TString(name); description.UserType = userType; description.TypeConfig = typeConfig; + description.LangVer = ctx.Types.LangVer; ctx.Types.Credentials->ForEach([&description](const TString& name, const TCredential& cred) { description.SecureParams[TString("token:") + name] = cred.Content; if (name.StartsWith("default_")) { @@ -7679,6 +7706,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> cached.NormalizedUserType = description.NormalizedUserType ? description.NormalizedUserType : ctx.Expr.MakeType<TVoidExprType>(); cached.SupportsBlocks = description.SupportsBlocks; cached.IsStrict = description.IsStrict; + cached.MinLangVer = description.MinLangVer; + cached.MaxLangVer = description.MaxLangVer; if (name != cached.NormalizedName) { ctx.Types.UdfTypeCache[std::make_tuple(cached.NormalizedName, TString(typeConfig), userType)] = cached; @@ -7733,6 +7762,20 @@ template <NKikimr::NUdf::EDataSlot DataSlot> .Seal(); } + if (cached.MinLangVer != UnknownLangVersion) { + parent.List(settingIndex++) + .Atom(0, "minLang") + .Atom(1, cached.MinLangVer) + .Seal(); + } + + if (cached.MaxLangVer != UnknownLangVersion) { + parent.List(settingIndex++) + .Atom(0, "maxLang") + .Atom(1, cached.MaxLangVer) + .Seal(); + } + if (settings) { if (auto setting = GetSetting(*settings, "cpu")) { parent.Add(settingIndex++, setting); diff --git a/yql/essentials/core/type_ann/ya.make b/yql/essentials/core/type_ann/ya.make index 80b44ce2799..a71db2a9c8e 100644 --- a/yql/essentials/core/type_ann/ya.make +++ b/yql/essentials/core/type_ann/ya.make @@ -36,6 +36,7 @@ PEERDIR( yql/essentials/parser/pg_catalog yql/essentials/core/sql_types yql/essentials/parser/pg_wrapper/interface + yql/essentials/public/langver library/cpp/yson/node ) diff --git a/yql/essentials/core/yql_type_annotation.h b/yql/essentials/core/yql_type_annotation.h index a43650a01f4..ba5d04fc170 100644 --- a/yql/essentials/core/yql_type_annotation.h +++ b/yql/essentials/core/yql_type_annotation.h @@ -339,6 +339,8 @@ struct TUdfCachedInfo { const TTypeAnnotationNode* NormalizedUserType = nullptr; bool SupportsBlocks = false; bool IsStrict = false; + TLangVersion MinLangVer = UnknownLangVersion; + TLangVersion MaxLangVer = UnknownLangVersion; }; const TString TypeAnnotationContextComponent = "TypeAnnotationContext"; diff --git a/yql/essentials/core/yql_udf_resolver.h b/yql/essentials/core/yql_udf_resolver.h index dff7e5bc07a..915a7a40407 100644 --- a/yql/essentials/core/yql_udf_resolver.h +++ b/yql/essentials/core/yql_udf_resolver.h @@ -4,6 +4,7 @@ #include <yql/essentials/providers/common/proto/udf_resolver.pb.h> #include <yql/essentials/core/yql_holding_file_storage.h> +#include <yql/essentials/public/langver/yql_langver.h> #include <yql/essentials/public/issue/yql_issue.h> #include <yql/essentials/public/udf/udf_log.h> @@ -42,6 +43,7 @@ public: TString TypeConfig; const TTypeAnnotationNode* UserType = nullptr; THashMap<TString, TString> SecureParams; + NYql::TLangVersion LangVer = NYql::UnknownLangVersion; // output TString NormalizedName; @@ -50,6 +52,8 @@ public: const TTypeAnnotationNode* CallableType = nullptr; bool SupportsBlocks = false; bool IsStrict = false; + NYql::TLangVersion MinLangVer = NYql::UnknownLangVersion; + NYql::TLangVersion MaxLangVer = NYql::UnknownLangVersion; TVector<TString> Messages; }; diff --git a/yql/essentials/minikql/comp_nodes/mkql_scalar_apply.cpp b/yql/essentials/minikql/comp_nodes/mkql_scalar_apply.cpp index 995fd76da74..28059ca4c3e 100644 --- a/yql/essentials/minikql/comp_nodes/mkql_scalar_apply.cpp +++ b/yql/essentials/minikql/comp_nodes/mkql_scalar_apply.cpp @@ -55,7 +55,7 @@ public: , TimeProvider(CreateDefaultTimeProvider()) , Ctx(HolderFactory, &ValueBuilder, TComputationOptsFull( nullptr, Alloc.Ref(), TypeEnv, *RandomProvider, *TimeProvider, NUdf::EValidatePolicy::Exception, originalContext.SecureParamsProvider, - originalContext.CountersProvider, originalContext.LogProvider), + originalContext.CountersProvider, originalContext.LogProvider, originalContext.LangVer), originalContext.Mutables, *NYql::NUdf::GetYqlMemoryPool()) { Alloc.Ref().EnableArrowTracking = false; diff --git a/yql/essentials/minikql/comp_nodes/mkql_udf.cpp b/yql/essentials/minikql/comp_nodes/mkql_udf.cpp index c74c282d272..579f4ab09c0 100644 --- a/yql/essentials/minikql/comp_nodes/mkql_udf.cpp +++ b/yql/essentials/minikql/comp_nodes/mkql_udf.cpp @@ -50,7 +50,7 @@ public: ui32 flags = 0; TFunctionTypeInfo funcInfo; const auto status = ctx.HolderFactory.GetFunctionRegistry()->FindFunctionTypeInfo( - ctx.TypeEnv, ctx.TypeInfoHelper, ctx.CountersProvider, FunctionName, UserType->IsVoid() ? nullptr : UserType, + ctx.LangVer, ctx.TypeEnv, ctx.TypeInfoHelper, ctx.CountersProvider, FunctionName, UserType->IsVoid() ? nullptr : UserType, TypeConfig, flags, Pos, ctx.SecureParamsProvider, ctx.LogProvider, &funcInfo); if (!status.IsOk()) { @@ -204,7 +204,7 @@ private: ui32 flags = 0; TFunctionTypeInfo funcInfo; const auto status = ctx.HolderFactory.GetFunctionRegistry()->FindFunctionTypeInfo( - ctx.TypeEnv, ctx.TypeInfoHelper, ctx.CountersProvider, FunctionName, UserType->IsVoid() ? nullptr : UserType, + ctx.LangVer, ctx.TypeEnv, ctx.TypeInfoHelper, ctx.CountersProvider, FunctionName, UserType->IsVoid() ? nullptr : UserType, TypeConfig, flags, Pos, ctx.SecureParamsProvider, ctx.LogProvider, &funcInfo); if (!status.IsOk()) { @@ -290,7 +290,7 @@ IComputationNode* WrapUdf(TCallable& callable, const TComputationNodeFactoryCont const auto userType = static_cast<TType*>(userTypeNode.GetNode()); const auto status = ctx.FunctionRegistry.FindFunctionTypeInfo( - ctx.Env, ctx.TypeInfoHelper, ctx.CountersProvider, funcName, userType->IsVoid() ? nullptr : userType, + ctx.LangVer, ctx.Env, ctx.TypeInfoHelper, ctx.CountersProvider, funcName, userType->IsVoid() ? nullptr : userType, typeConfig, flags, pos, ctx.SecureParamsProvider, ctx.LogProvider, &funcInfo); if (!status.IsOk()) { @@ -352,7 +352,7 @@ IComputationNode* WrapScriptUdf(TCallable& callable, const TComputationNodeFacto ui32 flags = 0; TFunctionTypeInfo funcInfo; const auto status = ctx.FunctionRegistry.FindFunctionTypeInfo( - ctx.Env, ctx.TypeInfoHelper, ctx.CountersProvider, funcName, userType, + ctx.LangVer, ctx.Env, ctx.TypeInfoHelper, ctx.CountersProvider, funcName, userType, typeConfig, flags, pos, ctx.SecureParamsProvider, ctx.LogProvider, &funcInfo); if (!status.IsOk()) { diff --git a/yql/essentials/minikql/computation/mkql_computation_node.h b/yql/essentials/minikql/computation/mkql_computation_node.h index 7238f1ed418..b1004629394 100644 --- a/yql/essentials/minikql/computation/mkql_computation_node.h +++ b/yql/essentials/minikql/computation/mkql_computation_node.h @@ -11,6 +11,7 @@ #include <yql/essentials/minikql/mkql_stats_registry.h> #include <yql/essentials/minikql/mkql_terminator.h> +#include <yql/essentials/public/langver/yql_langver.h> #include <yql/essentials/public/udf/udf_value.h> #include <yql/essentials/public/udf/udf_validate.h> #include <yql/essentials/public/udf/udf_value_builder.h> @@ -46,7 +47,7 @@ struct TComputationOpts { struct TComputationOptsFull: public TComputationOpts { TComputationOptsFull(IStatsRegistry* stats, TAllocState& allocState, const TTypeEnvironment& typeEnv, IRandomProvider& randomProvider, ITimeProvider& timeProvider, NUdf::EValidatePolicy validatePolicy, const NUdf::ISecureParamsProvider* secureParamsProvider, - NUdf::ICountersProvider* countersProvider, const NUdf::ILogProvider* logProvider) + NUdf::ICountersProvider* countersProvider, const NUdf::ILogProvider* logProvider, NYql::TLangVersion langver) : TComputationOpts(stats) , AllocState(allocState) , TypeEnv(typeEnv) @@ -56,6 +57,7 @@ struct TComputationOptsFull: public TComputationOpts { , SecureParamsProvider(secureParamsProvider) , CountersProvider(countersProvider) , LogProvider(logProvider) + , LangVer(langver) {} TAllocState& AllocState; @@ -66,6 +68,7 @@ struct TComputationOptsFull: public TComputationOpts { const NUdf::ISecureParamsProvider *const SecureParamsProvider; NUdf::ICountersProvider *const CountersProvider; const NUdf::ILogProvider* const LogProvider; + const NYql::TLangVersion LangVer; }; struct TWideFieldsInitInfo { @@ -121,6 +124,7 @@ struct TComputationContext : public TComputationContextLLVM { NUdf::ICountersProvider *const CountersProvider; const NUdf::ISecureParamsProvider *const SecureParamsProvider; const NUdf::ILogProvider* LogProvider; + NYql::TLangVersion LangVer = NYql::UnknownLangVersion; TComputationContext(const THolderFactory& holderFactory, const NUdf::IValueBuilder* builder, @@ -280,6 +284,7 @@ struct TComputationNodeFactoryContext { NUdf::ICountersProvider* CountersProvider; const NUdf::ISecureParamsProvider* SecureParamsProvider; const NUdf::ILogProvider* LogProvider; + NYql::TLangVersion LangVer; const TNodeFactory& NodeFactory; const THolderFactory& HolderFactory; const NUdf::IValueBuilder *const Builder; @@ -291,23 +296,24 @@ struct TComputationNodeFactoryContext { const TNodePushBack NodePushBack; TComputationNodeFactoryContext( - const TNodeLocator& nodeLocator, - const IFunctionRegistry& functionRegistry, - const TTypeEnvironment& env, - NUdf::ITypeInfoHelper::TPtr typeInfoHelper, - NUdf::ICountersProvider* countersProvider, - const NUdf::ISecureParamsProvider* secureParamsProvider, - const NUdf::ILogProvider* logProvider, - const TNodeFactory& nodeFactory, - const THolderFactory& holderFactory, - const NUdf::IValueBuilder* builder, - NUdf::EValidateMode validateMode, - NUdf::EValidatePolicy validatePolicy, - EGraphPerProcess graphPerProcess, - TComputationMutables& mutables, - TComputationNodeOnNodeMap& elementsCache, - TNodePushBack&& nodePushBack - ) + const TNodeLocator& nodeLocator, + const IFunctionRegistry& functionRegistry, + const TTypeEnvironment& env, + NUdf::ITypeInfoHelper::TPtr typeInfoHelper, + NUdf::ICountersProvider* countersProvider, + const NUdf::ISecureParamsProvider* secureParamsProvider, + const NUdf::ILogProvider* logProvider, + NYql::TLangVersion langver, + const TNodeFactory& nodeFactory, + const THolderFactory& holderFactory, + const NUdf::IValueBuilder* builder, + NUdf::EValidateMode validateMode, + NUdf::EValidatePolicy validatePolicy, + EGraphPerProcess graphPerProcess, + TComputationMutables& mutables, + TComputationNodeOnNodeMap& elementsCache, + TNodePushBack&& nodePushBack + ) : NodeLocator(nodeLocator) , FunctionRegistry(functionRegistry) , Env(env) @@ -315,6 +321,7 @@ struct TComputationNodeFactoryContext { , CountersProvider(countersProvider) , SecureParamsProvider(secureParamsProvider) , LogProvider(logProvider) + , LangVer(langver) , NodeFactory(nodeFactory) , HolderFactory(holderFactory) , Builder(builder) @@ -350,7 +357,8 @@ struct TComputationPatternOpts { IStatsRegistry* stats = nullptr, NUdf::ICountersProvider* countersProvider = nullptr, const NUdf::ISecureParamsProvider* secureParamsProvider = nullptr, - const NUdf::ILogProvider* logProvider = nullptr) + const NUdf::ILogProvider* logProvider = nullptr, + NYql::TLangVersion langver = NYql::UnknownLangVersion) : AllocState(allocState) , Env(env) , Factory(factory) @@ -363,6 +371,7 @@ struct TComputationPatternOpts { , CountersProvider(countersProvider) , SecureParamsProvider(secureParamsProvider) , LogProvider(logProvider) + , LangVer(langver) {} void SetOptions(TComputationNodeFactory factory, const IFunctionRegistry* functionRegistry, @@ -370,7 +379,7 @@ struct TComputationPatternOpts { const TString& optLLVM, EGraphPerProcess graphPerProcess, IStatsRegistry* stats = nullptr, NUdf::ICountersProvider* counters = nullptr, const NUdf::ISecureParamsProvider* secureParamsProvider = nullptr, - const NUdf::ILogProvider* logProvider = nullptr) { + const NUdf::ILogProvider* logProvider = nullptr, NYql::TLangVersion langver = NYql::UnknownLangVersion) { Factory = factory; FunctionRegistry = functionRegistry; ValidateMode = validateMode; @@ -381,6 +390,7 @@ struct TComputationPatternOpts { CountersProvider = counters; SecureParamsProvider = secureParamsProvider; LogProvider = logProvider; + LangVer = langver; } void SetPatternEnv(std::shared_ptr<TPatternCacheEntry> cacheEnv) { @@ -401,10 +411,11 @@ struct TComputationPatternOpts { NUdf::ICountersProvider* CountersProvider = nullptr; const NUdf::ISecureParamsProvider* SecureParamsProvider = nullptr; const NUdf::ILogProvider* LogProvider = nullptr; + NYql::TLangVersion LangVer = NYql::UnknownLangVersion; TComputationOptsFull ToComputationOptions(IRandomProvider& randomProvider, ITimeProvider& timeProvider, TAllocState* allocStatePtr = nullptr) const { return TComputationOptsFull(Stats, allocStatePtr ? *allocStatePtr : AllocState, Env, randomProvider, timeProvider, - ValidatePolicy, SecureParamsProvider, CountersProvider, LogProvider); + ValidatePolicy, SecureParamsProvider, CountersProvider, LogProvider, LangVer); } }; diff --git a/yql/essentials/minikql/computation/mkql_computation_node_graph.cpp b/yql/essentials/minikql/computation/mkql_computation_node_graph.cpp index 9f2e182acc8..c2c26a2cd79 100644 --- a/yql/essentials/minikql/computation/mkql_computation_node_graph.cpp +++ b/yql/essentials/minikql/computation/mkql_computation_node_graph.cpp @@ -239,6 +239,7 @@ public: , CountersProvider(opts.CountersProvider) , SecureParamsProvider(opts.SecureParamsProvider) , LogProvider(opts.LogProvider) + , LangVer(opts.LangVer) , Factory(opts.Factory) , FunctionRegistry(*opts.FunctionRegistry) , ValidateMode(opts.ValidateMode) @@ -465,6 +466,7 @@ private: CountersProvider, SecureParamsProvider, LogProvider, + LangVer, *NodeFactory, *PatternNodes->HolderFactory, PatternNodes->ValueBuilder.Get(), @@ -562,6 +564,7 @@ private: NUdf::ICountersProvider* CountersProvider; const NUdf::ISecureParamsProvider* SecureParamsProvider; const NUdf::ILogProvider* LogProvider; + const NYql::TLangVersion LangVer; const TComputationNodeFactory Factory; const IFunctionRegistry& FunctionRegistry; TIntrusivePtr<TMemoryUsageInfo> MemInfo; diff --git a/yql/essentials/minikql/computation/mkql_validate_ut.cpp b/yql/essentials/minikql/computation/mkql_validate_ut.cpp index 3a1d0588ff6..b4e4d50670a 100644 --- a/yql/essentials/minikql/computation/mkql_validate_ut.cpp +++ b/yql/essentials/minikql/computation/mkql_validate_ut.cpp @@ -683,7 +683,8 @@ Y_UNIT_TEST_SUITE(TMiniKQLValidateTest) { TFunctionTypeInfo funcInfo; TType* userType = nullptr; TStringBuf typeConfig; - TStatus status = functionRegistry->FindFunctionTypeInfo(env, typeInfoHelper, nullptr, udfFuncName, userType, typeConfig, flags, {}, nullptr, nullptr, &funcInfo); + TStatus status = functionRegistry->FindFunctionTypeInfo(NYql::UnknownLangVersion, env, typeInfoHelper, + nullptr, udfFuncName, userType, typeConfig, flags, {}, nullptr, nullptr, &funcInfo); MKQL_ENSURE(status.IsOk(), status.GetError()); auto type = funcInfo.FunctionType->GetReturnType(); fullValidateFunc(value, builder, type); @@ -733,7 +734,8 @@ Y_UNIT_TEST_SUITE(TMiniKQLValidateTest) { TType* userType = nullptr; TStringBuf typeConfig; NUdf::ITypeInfoHelper::TPtr typeInfoHelper(new TTypeInfoHelper); - TStatus status = functionRegistry.FindFunctionTypeInfo(pgmBuilder.GetTypeEnvironment(), typeInfoHelper, nullptr, + TStatus status = functionRegistry.FindFunctionTypeInfo(NYql::UnknownLangVersion, + pgmBuilder.GetTypeEnvironment(), typeInfoHelper, nullptr, udfFuncName, userType, typeConfig, flags, {}, nullptr, nullptr, &funcInfo); MKQL_ENSURE(status.IsOk(), status.GetError()); auto callable = pgmBuilder.Udf(udfFuncName); diff --git a/yql/essentials/minikql/computation/mkql_value_builder_ut.cpp b/yql/essentials/minikql/computation/mkql_value_builder_ut.cpp index 4d1e90a950e..127c55bc9b2 100644 --- a/yql/essentials/minikql/computation/mkql_value_builder_ut.cpp +++ b/yql/essentials/minikql/computation/mkql_value_builder_ut.cpp @@ -44,7 +44,7 @@ public: , HolderFactory(Alloc.Ref(), MemInfo, FunctionRegistry.Get()) , Builder(HolderFactory, NUdf::EValidatePolicy::Exception) , TypeInfoHelper(new TTypeInfoHelper()) - , FunctionTypeInfoBuilder(Env, TypeInfoHelper, "", nullptr, {}) + , FunctionTypeInfoBuilder(NYql::UnknownLangVersion, Env, TypeInfoHelper, "", nullptr, {}) { BoolOid = NYql::NPg::LookupType("bool").TypeId; } diff --git a/yql/essentials/minikql/computation/ya.make.inc b/yql/essentials/minikql/computation/ya.make.inc index 14f51bfceed..5f7cdd929bd 100644 --- a/yql/essentials/minikql/computation/ya.make.inc +++ b/yql/essentials/minikql/computation/ya.make.inc @@ -22,6 +22,7 @@ PEERDIR( yql/essentials/minikql/arrow yql/essentials/minikql/computation yql/essentials/parser/pg_wrapper/interface + yql/essentials/public/langver yql/essentials/public/udf yql/essentials/public/udf/arrow yql/essentials/utils diff --git a/yql/essentials/minikql/mkql_function_registry.cpp b/yql/essentials/minikql/mkql_function_registry.cpp index 27933e682d8..355b5a4e7e8 100644 --- a/yql/essentials/minikql/mkql_function_registry.cpp +++ b/yql/essentials/minikql/mkql_function_registry.cpp @@ -239,23 +239,24 @@ public: } TStatus FindFunctionTypeInfo( - const TTypeEnvironment& env, - NUdf::ITypeInfoHelper::TPtr typeInfoHelper, - NUdf::ICountersProvider* countersProvider, - const TStringBuf& name, - TType* userType, - const TStringBuf& typeConfig, - ui32 flags, - const NUdf::TSourcePosition& pos, - const NUdf::ISecureParamsProvider* secureParamsProvider, - const NUdf::ILogProvider* logProvider, - TFunctionTypeInfo* funcInfo) const override + NYql::TLangVersion langver, + const TTypeEnvironment& env, + NUdf::ITypeInfoHelper::TPtr typeInfoHelper, + NUdf::ICountersProvider* countersProvider, + const TStringBuf& name, + TType* userType, + const TStringBuf& typeConfig, + ui32 flags, + const NUdf::TSourcePosition& pos, + const NUdf::ISecureParamsProvider* secureParamsProvider, + const NUdf::ILogProvider* logProvider, + TFunctionTypeInfo* funcInfo) const override { TStringBuf moduleName, funcName; if (name.TrySplit(MODULE_NAME_DELIMITER, moduleName, funcName)) { auto it = UdfModules_.find(moduleName); if (it != UdfModules_.end()) { - TFunctionTypeInfoBuilder typeInfoBuilder(env, typeInfoHelper, moduleName, + TFunctionTypeInfoBuilder typeInfoBuilder(langver, env, typeInfoHelper, moduleName, (flags & NUdf::IUdfModule::TFlags::TypesOnly) ? nullptr : countersProvider, pos, secureParamsProvider, logProvider); const auto& module = *it->second.Impl; @@ -412,18 +413,20 @@ public: } TStatus FindFunctionTypeInfo( - const TTypeEnvironment& env, - NUdf::ITypeInfoHelper::TPtr typeInfoHelper, - NUdf::ICountersProvider* countersProvider, - const TStringBuf& name, - TType* userType, - const TStringBuf& typeConfig, - ui32 flags, - const NUdf::TSourcePosition& pos, - const NUdf::ISecureParamsProvider* secureParamsProvider, - const NUdf::ILogProvider* logProvider, - TFunctionTypeInfo* funcInfo) const override + NYql::TLangVersion langver, + const TTypeEnvironment& env, + NUdf::ITypeInfoHelper::TPtr typeInfoHelper, + NUdf::ICountersProvider* countersProvider, + const TStringBuf& name, + TType* userType, + const TStringBuf& typeConfig, + ui32 flags, + const NUdf::TSourcePosition& pos, + const NUdf::ISecureParamsProvider* secureParamsProvider, + const NUdf::ILogProvider* logProvider, + TFunctionTypeInfo* funcInfo) const override { + Y_UNUSED(langver); Y_UNUSED(env); Y_UNUSED(typeInfoHelper); Y_UNUSED(countersProvider); diff --git a/yql/essentials/minikql/mkql_function_registry.h b/yql/essentials/minikql/mkql_function_registry.h index bdf2e663d5f..5b648a26e8c 100644 --- a/yql/essentials/minikql/mkql_function_registry.h +++ b/yql/essentials/minikql/mkql_function_registry.h @@ -2,6 +2,7 @@ #include "mkql_function_metadata.h" +#include <yql/essentials/public/langver/yql_langver.h> #include <yql/essentials/public/udf/udf_counter.h> #include <yql/essentials/public/udf/udf_registrator.h> #include <yql/essentials/public/udf/udf_type_builder.h> @@ -60,17 +61,18 @@ public: virtual void AllowUdfPatch() = 0; virtual TStatus FindFunctionTypeInfo( - const TTypeEnvironment& env, - NUdf::ITypeInfoHelper::TPtr typeInfoHelper, - NUdf::ICountersProvider* countersProvider, - const TStringBuf& name, - TType* userType, - const TStringBuf& typeConfig, - ui32 flags, - const NUdf::TSourcePosition& pos, - const NUdf::ISecureParamsProvider* secureParamsProvider, - const NUdf::ILogProvider* logProvider, - TFunctionTypeInfo* funcInfo) const = 0; + NYql::TLangVersion langver, + const TTypeEnvironment& env, + NUdf::ITypeInfoHelper::TPtr typeInfoHelper, + NUdf::ICountersProvider* countersProvider, + const TStringBuf& name, + TType* userType, + const TStringBuf& typeConfig, + ui32 flags, + const NUdf::TSourcePosition& pos, + const NUdf::ISecureParamsProvider* secureParamsProvider, + const NUdf::ILogProvider* logProvider, + TFunctionTypeInfo* funcInfo) const = 0; virtual TMaybe<TString> FindUdfPath(const TStringBuf& moduleName) const = 0; diff --git a/yql/essentials/minikql/mkql_program_builder.cpp b/yql/essentials/minikql/mkql_program_builder.cpp index 97d681db74e..8bed557c701 100644 --- a/yql/essentials/minikql/mkql_program_builder.cpp +++ b/yql/essentials/minikql/mkql_program_builder.cpp @@ -338,10 +338,12 @@ std::vector<TType*> ValidateBlockFlowType(const TType* flowType, bool unwrap) { return ValidateBlockItems(wideComponents, unwrap); } -TProgramBuilder::TProgramBuilder(const TTypeEnvironment& env, const IFunctionRegistry& functionRegistry, bool voidWithEffects) +TProgramBuilder::TProgramBuilder(const TTypeEnvironment& env, const IFunctionRegistry& functionRegistry, + bool voidWithEffects, NYql::TLangVersion langver) : TTypeBuilder(env) , FunctionRegistry(functionRegistry) , VoidWithEffects(voidWithEffects) + , LangVer(langver) {} const TTypeEnvironment& TProgramBuilder::GetTypeEnvironment() const { @@ -4097,7 +4099,7 @@ TRuntimeNode TProgramBuilder::Udf( TFunctionTypeInfo funcInfo; TStatus status = FunctionRegistry.FindFunctionTypeInfo( - Env, TypeInfoHelper, nullptr, funcName, userType, typeConfig, flags, {}, nullptr, nullptr, &funcInfo); + LangVer, Env, TypeInfoHelper, nullptr, funcName, userType, typeConfig, flags, {}, nullptr, nullptr, &funcInfo); MKQL_ENSURE(status.IsOk(), status.GetError()); auto runConfigType = funcInfo.RunConfigType; diff --git a/yql/essentials/minikql/mkql_program_builder.h b/yql/essentials/minikql/mkql_program_builder.h index e801d97da44..67a74dd48b6 100644 --- a/yql/essentials/minikql/mkql_program_builder.h +++ b/yql/essentials/minikql/mkql_program_builder.h @@ -4,6 +4,7 @@ #include "mkql_node.h" #include "mkql_node_builder.h" #include "mkql_type_builder.h" +#include <yql/essentials/public/langver/yql_langver.h> #include <yql/essentials/public/udf/udf_value.h> #include <yql/essentials/core/sql_types/match_recognize.h> @@ -142,7 +143,8 @@ std::vector<TType*> ValidateBlockFlowType(const TType* flowType, bool unwrap = t class TProgramBuilder : public TTypeBuilder { public: - TProgramBuilder(const TTypeEnvironment& env, const IFunctionRegistry& functionRegistry, bool voidWithEffects = false); + TProgramBuilder(const TTypeEnvironment& env, const IFunctionRegistry& functionRegistry, bool voidWithEffects = false, + NYql::TLangVersion langver = NYql::UnknownLangVersion); const TTypeEnvironment& GetTypeEnvironment() const; const IFunctionRegistry& GetFunctionRegistry() const; @@ -868,6 +870,7 @@ private: protected: const IFunctionRegistry& FunctionRegistry; const bool VoidWithEffects; + const NYql::TLangVersion LangVer; NUdf::ITypeInfoHelper::TPtr TypeInfoHelper; }; diff --git a/yql/essentials/minikql/mkql_type_builder.cpp b/yql/essentials/minikql/mkql_type_builder.cpp index 41502b144ae..59c00d17ebc 100644 --- a/yql/essentials/minikql/mkql_type_builder.cpp +++ b/yql/essentials/minikql/mkql_type_builder.cpp @@ -1716,6 +1716,7 @@ void TArrowType::Export(ArrowSchema* out) const { // TFunctionTypeInfoBuilder ////////////////////////////////////////////////////////////////////////////// TFunctionTypeInfoBuilder::TFunctionTypeInfoBuilder( + NYql::TLangVersion langver, const TTypeEnvironment& env, NUdf::ITypeInfoHelper::TPtr typeInfoHelper, const TStringBuf& moduleName, @@ -1723,7 +1724,8 @@ TFunctionTypeInfoBuilder::TFunctionTypeInfoBuilder( const NUdf::TSourcePosition& pos, const NUdf::ISecureParamsProvider* secureParamsProvider, const NUdf::ILogProvider* logProvider) - : Env_(env) + : LangVer_(langver) + , Env_(env) , ReturnType_(nullptr) , RunConfigType_(Env_.GetTypeOfVoidLazy()) , UserType_(Env_.GetTypeOfVoidLazy()) @@ -1830,6 +1832,18 @@ NUdf::TLoggerPtr TFunctionTypeInfoBuilder::MakeLogger(bool synchronized) const { return ret; } +void TFunctionTypeInfoBuilder::SetMinLangVer(ui32 langver) { + MinLangVer_ = langver; + +} +void TFunctionTypeInfoBuilder::SetMaxLangVer(ui32 langver) { + MaxLangVer_ = langver; +} + +ui32 TFunctionTypeInfoBuilder::GetCurrentLangVer() const { + return LangVer_; +} + NUdf::IFunctionTypeInfoBuilder1& TFunctionTypeInfoBuilder::ReturnsImpl( NUdf::TDataTypeId typeId) { @@ -1948,6 +1962,8 @@ void TFunctionTypeInfoBuilder::Build(TFunctionTypeInfo* funcInfo) funcInfo->IRFunctionName = std::move(IRFunctionName_); funcInfo->SupportsBlocks = SupportsBlocks_; funcInfo->IsStrict = IsStrict_; + funcInfo->MinLangVer = MinLangVer_; + funcInfo->MaxLangVer = MaxLangVer_; } NUdf::TType* TFunctionTypeInfoBuilder::Primitive(NUdf::TDataTypeId typeId) const diff --git a/yql/essentials/minikql/mkql_type_builder.h b/yql/essentials/minikql/mkql_type_builder.h index 678db9157ca..38f2869c81e 100644 --- a/yql/essentials/minikql/mkql_type_builder.h +++ b/yql/essentials/minikql/mkql_type_builder.h @@ -3,8 +3,10 @@ #include "mkql_node.h" #include <yql/essentials/core/sql_types/block.h> +#include <yql/essentials/public/langver/yql_langver.h> #include <yql/essentials/public/udf/udf_type_builder.h> #include <yql/essentials/public/udf/arrow/block_type_helper.h> +#include <yql/essentials/public/langver/yql_langver.h> #include <yql/essentials/parser/pg_wrapper/interface/compare.h> #include <util/generic/size_literals.h> @@ -103,6 +105,8 @@ struct TFunctionTypeInfo bool Deterministic = true; bool SupportsBlocks = false; bool IsStrict = false; + NYql::TLangVersion MinLangVer = NYql::UnknownLangVersion; + NYql::TLangVersion MaxLangVer = NYql::UnknownLangVersion; }; ////////////////////////////////////////////////////////////////////////////// @@ -121,13 +125,14 @@ class TFunctionTypeInfoBuilder: public NUdf::IFunctionTypeInfoBuilder { public: TFunctionTypeInfoBuilder( - const TTypeEnvironment& env, - NUdf::ITypeInfoHelper::TPtr typeInfoHelper, - const TStringBuf& moduleName, - NUdf::ICountersProvider* countersProvider, - const NUdf::TSourcePosition& pos, - const NUdf::ISecureParamsProvider* secureParamsProvider = nullptr, - const NUdf::ILogProvider* logProvider = nullptr); + NYql::TLangVersion langver, + const TTypeEnvironment& env, + NUdf::ITypeInfoHelper::TPtr typeInfoHelper, + const TStringBuf& moduleName, + NUdf::ICountersProvider* countersProvider, + const NUdf::TSourcePosition& pos, + const NUdf::ISecureParamsProvider* secureParamsProvider = nullptr, + const NUdf::ILogProvider* logProvider = nullptr); NUdf::IFunctionTypeInfoBuilder1& ImplementationImpl( NUdf::TUniquePtr<NUdf::IBoxedValue> impl) override; @@ -209,8 +214,12 @@ public: bool GetSecureParam(NUdf::TStringRef key, NUdf::TStringRef& value) const override; NUdf::TLoggerPtr MakeLogger(bool synchronized) const override; + void SetMinLangVer(ui32 langver) override; + void SetMaxLangVer(ui32 langver) override; + ui32 GetCurrentLangVer() const override; private: + const NYql::TLangVersion LangVer_; const TTypeEnvironment& Env_; NUdf::TUniquePtr<NUdf::IBoxedValue> Implementation_; const TType* ReturnType_; @@ -232,6 +241,8 @@ private: TString IRFunctionName_; bool SupportsBlocks_ = false; bool IsStrict_ = false; + ui32 MinLangVer_ = NYql::UnknownLangVersion; + ui32 MaxLangVer_ = NYql::UnknownLangVersion; }; class TTypeInfoHelper : public NUdf::ITypeInfoHelper diff --git a/yql/essentials/minikql/mkql_type_builder_ut.cpp b/yql/essentials/minikql/mkql_type_builder_ut.cpp index 3c84c6e05d6..95d47795df7 100644 --- a/yql/essentials/minikql/mkql_type_builder_ut.cpp +++ b/yql/essentials/minikql/mkql_type_builder_ut.cpp @@ -16,7 +16,7 @@ public: : Alloc(__LOCATION__) , Env(Alloc) , TypeInfoHelper(new TTypeInfoHelper()) - , FunctionTypeInfoBuilder(Env, TypeInfoHelper, "", nullptr, {}) { + , FunctionTypeInfoBuilder(NYql::UnknownLangVersion, Env, TypeInfoHelper, "", nullptr, {}) { } private: @@ -369,7 +369,7 @@ Y_UNIT_TEST_SUITE(TLogProviderTest) { [this](const NUdf::TStringRef& component, NUdf::ELogLevel level, const NUdf::TStringRef& message) { Messages.push_back({TString(component), level, TString(message)}); })) - , FunctionTypeInfoBuilder(Env, TypeInfoHelper, "module", nullptr, {}, nullptr, withoutLog ? nullptr : LogProvider.Get()) + , FunctionTypeInfoBuilder(NYql::UnknownLangVersion, Env, TypeInfoHelper, "module", nullptr, {}, nullptr, withoutLog ? nullptr : LogProvider.Get()) {} TScopedAlloc Alloc; diff --git a/yql/essentials/minikql/protobuf_udf/ut/type_builder_ut.cpp b/yql/essentials/minikql/protobuf_udf/ut/type_builder_ut.cpp index 4e903f43a05..4782d5aae99 100644 --- a/yql/essentials/minikql/protobuf_udf/ut/type_builder_ut.cpp +++ b/yql/essentials/minikql/protobuf_udf/ut/type_builder_ut.cpp @@ -31,7 +31,7 @@ struct TSetup { , Env(Alloc) , FunctionRegistry(CreateFunctionRegistry(IBuiltinFunctionRegistry::TPtr())) , TypeInfoHelper(new TTypeInfoHelper()) - , FunctionTypeInfoBuilder(Env, TypeInfoHelper, "", nullptr, {}) + , FunctionTypeInfoBuilder(UnknownLangVersion, Env, TypeInfoHelper, "", nullptr, {}) , PgmBuilder(Env, *FunctionRegistry) { } diff --git a/yql/essentials/minikql/protobuf_udf/ut/value_builder_ut.cpp b/yql/essentials/minikql/protobuf_udf/ut/value_builder_ut.cpp index a3ed5f796b8..1f79035f971 100644 --- a/yql/essentials/minikql/protobuf_udf/ut/value_builder_ut.cpp +++ b/yql/essentials/minikql/protobuf_udf/ut/value_builder_ut.cpp @@ -37,7 +37,7 @@ struct TSetup { , Env(Alloc) , FunctionRegistry(CreateFunctionRegistry(IBuiltinFunctionRegistry::TPtr())) , TypeInfoHelper(new TTypeInfoHelper()) - , FunctionTypeInfoBuilder(Env, TypeInfoHelper, "", nullptr, {}) + , FunctionTypeInfoBuilder(UnknownLangVersion, Env, TypeInfoHelper, "", nullptr, {}) , PgmBuilder(Env, *FunctionRegistry) , MemInfo("Test") , HolderFactory(Alloc.Ref(), MemInfo) diff --git a/yql/essentials/minikql/ya.make b/yql/essentials/minikql/ya.make index c1d03fb5875..4c09b76c66f 100644 --- a/yql/essentials/minikql/ya.make +++ b/yql/essentials/minikql/ya.make @@ -72,6 +72,7 @@ PEERDIR( yql/essentials/parser/pg_catalog yql/essentials/parser/pg_wrapper/interface yql/essentials/public/issue + yql/essentials/public/langver yql/essentials/public/udf yql/essentials/public/udf/tz yql/essentials/types/binary_json diff --git a/yql/essentials/providers/common/proto/udf_resolver.proto b/yql/essentials/providers/common/proto/udf_resolver.proto index 8693e9de4e5..b827d38dbfe 100644 --- a/yql/essentials/providers/common/proto/udf_resolver.proto +++ b/yql/essentials/providers/common/proto/udf_resolver.proto @@ -5,6 +5,7 @@ message TFunction { required string Name = 1; required string TypeConfig = 2; optional string UserType = 3; + optional uint32 LangVer = 4; }; message TArgResult { @@ -30,6 +31,8 @@ message TFunctionResult { optional bool SupportsBlocks = 14; optional bool IsStrict = 15; repeated string Messages = 16; + optional uint32 MinLangVer = 17; + optional uint32 MaxLangVer = 18; }; message TImport { diff --git a/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp b/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp index 9858c6ee256..e598b2d370f 100644 --- a/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp +++ b/yql/essentials/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp @@ -218,6 +218,8 @@ public: if (udf->UserType) { udfRequest->SetUserType(WriteTypeToYson(udf->UserType)); } + + udfRequest->SetLangVer(udf->LangVer); } TResolveResult response; @@ -368,6 +370,8 @@ private: } udf->SupportsBlocks = udfRes.GetSupportsBlocks(); udf->IsStrict = udfRes.GetIsStrict(); + udf->MinLangVer = udfRes.GetMinLangVer(); + udf->MaxLangVer = udfRes.GetMaxLangVer(); for (const auto& m : udfRes.GetMessages()) { udf->Messages.push_back(m); } diff --git a/yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.cpp b/yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.cpp index 8b19fdc1d85..fc9304bd4fc 100644 --- a/yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.cpp +++ b/yql/essentials/providers/common/udf_resolve/yql_simple_udf_resolver.cpp @@ -211,7 +211,7 @@ bool LoadFunctionsMetadata(const TVector<IUdfResolver::TFunction*>& functions, logLevel); TFunctionTypeInfo funcInfo; - auto status = functionRegistry.FindFunctionTypeInfo(env, typeInfoHelper, nullptr, + auto status = functionRegistry.FindFunctionTypeInfo(udf.LangVer, env, typeInfoHelper, nullptr, udf.Name, mkqlUserType, udf.TypeConfig, NUdf::IUdfModule::TFlags::TypesOnly, {}, secureParamsProvider.get(), logProvider.Get(), &funcInfo); if (!status.IsOk()) { @@ -236,6 +236,8 @@ bool LoadFunctionsMetadata(const TVector<IUdfResolver::TFunction*>& functions, udf.SupportsBlocks = funcInfo.SupportsBlocks; udf.IsStrict = funcInfo.IsStrict; + udf.MinLangVer = funcInfo.MinLangVer; + udf.MaxLangVer = funcInfo.MaxLangVer; } catch (const std::exception& e) { auto issue = TIssue(udf.Pos, TStringBuilder() << "Internal error was found when udf metadata is loading for function: " << udf.Name diff --git a/yql/essentials/providers/pure/yql_pure_provider.cpp b/yql/essentials/providers/pure/yql_pure_provider.cpp index e5ddb87bc1f..1446b8aa380 100644 --- a/yql/essentials/providers/pure/yql_pure_provider.cpp +++ b/yql/essentials/providers/pure/yql_pure_provider.cpp @@ -116,7 +116,7 @@ public: TScopedAlloc alloc(__LOCATION__, TAlignedPagePoolCounters(), State_->FunctionRegistry->SupportsSizedAllocators()); TTypeEnvironment env(alloc); - TProgramBuilder pgmBuilder(env, *State_->FunctionRegistry); + TProgramBuilder pgmBuilder(env, *State_->FunctionRegistry, false, State_->Types->LangVer); NCommon::TMkqlCommonCallableCompiler compiler; NCommon::TMkqlBuildContext mkqlCtx(compiler, pgmBuilder, ctx); @@ -145,7 +145,7 @@ public: auto pattern = MakeComputationPattern(explorer, root, {}, patternOpts); const TComputationOptsFull computeOpts(nullptr, alloc.Ref(), env, *State_->Types->RandomProvider, *State_->Types->TimeProvider, - NUdf::EValidatePolicy::Exception, nullptr, nullptr, logProvider.Get()); + NUdf::EValidatePolicy::Exception, nullptr, nullptr, logProvider.Get(), State_->Types->LangVer); auto graph = pattern->Clone(computeOpts); const TBindTerminator bind(graph->GetTerminator()); graph->Prepare(); diff --git a/yql/essentials/public/udf/udf_type_builder.h b/yql/essentials/public/udf/udf_type_builder.h index b5edff46eaf..9507b4bfe3b 100644 --- a/yql/essentials/public/udf/udf_type_builder.h +++ b/yql/essentials/public/udf/udf_type_builder.h @@ -647,7 +647,20 @@ public: }; #endif -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 42) +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 43) +class IFunctionTypeInfoBuilder18: public IFunctionTypeInfoBuilder17 { +public: + virtual void SetMinLangVer(ui32 langver) = 0; + + virtual void SetMaxLangVer(ui32 langver) = 0; + + virtual ui32 GetCurrentLangVer() const = 0; +}; +#endif + +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 43) +using IFunctionTypeInfoBuilderImpl = IFunctionTypeInfoBuilder18; +#elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 42) using IFunctionTypeInfoBuilderImpl = IFunctionTypeInfoBuilder17; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 32) using IFunctionTypeInfoBuilderImpl = IFunctionTypeInfoBuilder16; diff --git a/yql/essentials/public/udf/udf_version.h b/yql/essentials/public/udf/udf_version.h index 7b77f905431..78aaa7e3404 100644 --- a/yql/essentials/public/udf/udf_version.h +++ b/yql/essentials/public/udf/udf_version.h @@ -7,7 +7,7 @@ namespace NYql { namespace NUdf { #define CURRENT_UDF_ABI_VERSION_MAJOR 2 -#define CURRENT_UDF_ABI_VERSION_MINOR 42 +#define CURRENT_UDF_ABI_VERSION_MINOR 43 #define CURRENT_UDF_ABI_VERSION_PATCH 0 #ifdef USE_CURRENT_UDF_ABI_VERSION diff --git a/yql/essentials/tests/common/udf_test/test.py b/yql/essentials/tests/common/udf_test/test.py index 95b8cf1e18a..77c71f687e4 100644 --- a/yql/essentials/tests/common/udf_test/test.py +++ b/yql/essentials/tests/common/udf_test/test.py @@ -15,6 +15,7 @@ project_path = yatest.common.context.project_path SOURCE_PATH = yql_utils.yql_source_path((project_path + '/cases').replace('\\', '/')) DATA_PATH = yatest.common.output_path('cases') ASTDIFF_PATH = yql_utils.yql_binary_path(os.getenv('YQL_ASTDIFF_PATH') or 'yql/essentials/tools/astdiff/astdiff') +DEFAULT_LANG_VER = '2025.01' def pytest_generate_tests(metafunc): @@ -77,6 +78,9 @@ def test(case): xfail = yql_utils.is_xfail(cfg) if yql_utils.get_param('TARGET_PLATFORM') and xfail: pytest.skip('xfail is not supported on non-default target platform') + langver = yql_utils.get_langver(cfg) + if langver is None: + langver = DEFAULT_LANG_VER extra_env = dict(os.environ) extra_env["YQL_UDF_RESOLVER"] = "1" @@ -88,7 +92,11 @@ def test(case): if "YA_TEST_RUNNER" in extra_env: del extra_env["YA_TEST_RUNNER"] - yqlrun_res = YQLRun(udfs_dir=udfs_dir, prov='yt', use_sql2yql=False, cfg_dir=os.getenv('YQL_CONFIG_DIR') or 'yql/essentials/cfg/udf_test').yql_exec( + yqlrun = YQLRun(udfs_dir=udfs_dir, + prov='yt', use_sql2yql=False, + cfg_dir=os.getenv('YQL_CONFIG_DIR') or 'yql/essentials/cfg/udf_test', + langver=langver) + yqlrun_res = yqlrun.yql_exec( program=program, run_sql=True, tables=in_tables, diff --git a/yql/essentials/tools/udf_resolver/discover.cpp b/yql/essentials/tools/udf_resolver/discover.cpp index a290917a5a9..530e7d96e98 100644 --- a/yql/essentials/tools/udf_resolver/discover.cpp +++ b/yql/essentials/tools/udf_resolver/discover.cpp @@ -66,7 +66,7 @@ NYql::TResolveResult DoDiscover(const NYql::TResolve& inMsg, IMutableFunctionReg TFunctionTypeInfo funcInfo; if (!f.second.IsTypeAwareness) { - auto status = functionRegistry.FindFunctionTypeInfo(env, typeInfoHelper, + auto status = functionRegistry.FindFunctionTypeInfo(NYql::UnknownLangVersion, env, typeInfoHelper, nullptr, funcName, nullptr, nullptr, NUdf::IUdfModule::TFlags::TypesOnly, {}, nullptr, logProvider.Get(), &funcInfo); if (!status.IsOk()) { @@ -103,7 +103,8 @@ void Print(const NYql::TResolveResult& result, IOutputStream& out, bool printAsP out << "UDF count: " << result.UdfsSize() << Endl; } -void DiscoverInFiles(const TVector<TString>& udfPaths, IOutputStream& out, bool printAsProto, NYql::NUdf::ELogLevel logLevel) { +void DiscoverInFiles(const TVector<TString>& udfPaths, IOutputStream& out, bool printAsProto, + NYql::NUdf::ELogLevel logLevel) { NYql::TResolve inMsg; inMsg.SetRuntimeLogLevel(static_cast<ui32>(logLevel)); for (auto& path : udfPaths) { diff --git a/yql/essentials/tools/udf_resolver/discover.h b/yql/essentials/tools/udf_resolver/discover.h index 7c995b7f656..d32d3fa5a5a 100644 --- a/yql/essentials/tools/udf_resolver/discover.h +++ b/yql/essentials/tools/udf_resolver/discover.h @@ -9,8 +9,10 @@ #include <util/stream/output.h> namespace NUdfResolver { -void DiscoverInDir(const TString& dir, IOutputStream& out, bool printAsProto, NYql::NUdf::ELogLevel logLevel); -void DiscoverInFile(const TString& filePath, IOutputStream& out, bool printAsProto, NYql::NUdf::ELogLevel logLevel); +void DiscoverInDir(const TString& dir, IOutputStream& out, bool printAsProto, + NYql::NUdf::ELogLevel logLevel); +void DiscoverInFile(const TString& filePath, IOutputStream& out, bool printAsProto, + NYql::NUdf::ELogLevel logLevel); void Discover(IInputStream& in, IOutputStream& out, bool printAsProto); void FillImportResultModules(const THashSet<TString>& modules, NYql::TImportResult& importRes); ; diff --git a/yql/essentials/tools/udf_resolver/udf_resolver.cpp b/yql/essentials/tools/udf_resolver/udf_resolver.cpp index 27ab6f87d48..03116858f7e 100644 --- a/yql/essentials/tools/udf_resolver/udf_resolver.cpp +++ b/yql/essentials/tools/udf_resolver/udf_resolver.cpp @@ -174,7 +174,7 @@ void ResolveUDFs() { } TFunctionTypeInfo funcInfo; - auto status = newRegistry->FindFunctionTypeInfo(env, typeInfoHelper, nullptr, + auto status = newRegistry->FindFunctionTypeInfo(udf.GetLangVer(), env, typeInfoHelper, nullptr, udf.GetName(), mkqlUserType, udf.GetTypeConfig(), NUdf::IUdfModule::TFlags::TypesOnly, {}, nullptr, logProvider.Get(), &funcInfo); if (!status.IsOk()) { udfRes->SetError(TStringBuilder() << "Failed to find UDF function: " << udf.GetName() diff --git a/yql/essentials/tools/udf_resolver/ya.make b/yql/essentials/tools/udf_resolver/ya.make index 05240ec3b7b..2dff7abf78f 100644 --- a/yql/essentials/tools/udf_resolver/ya.make +++ b/yql/essentials/tools/udf_resolver/ya.make @@ -25,6 +25,7 @@ PEERDIR( yql/essentials/utils/backtrace yql/essentials/utils/sys yql/essentials/sql/pg_dummy + yql/essentials/public/langver ) YQL_LAST_ABI_VERSION() diff --git a/yql/essentials/udfs/common/python/bindings/py_test_engine.h b/yql/essentials/udfs/common/python/bindings/py_test_engine.h index c86febf779d..ff0a61cbb8a 100644 --- a/yql/essentials/udfs/common/python/bindings/py_test_engine.h +++ b/yql/essentials/udfs/common/python/bindings/py_test_engine.h @@ -46,7 +46,7 @@ public: , Alloc_(__LOCATION__) , Env_(Alloc_) , TypeInfoHelper_(new TTypeInfoHelper) - , FunctionInfoBuilder_(Env_, TypeInfoHelper_, "", nullptr, {}) + , FunctionInfoBuilder_(NYql::UnknownLangVersion, Env_, TypeInfoHelper_, "", nullptr, {}) { HolderFactory_ = MakeHolder<THolderFactory>( Alloc_.Ref(), diff --git a/yql/essentials/udfs/common/string/string_udf.cpp b/yql/essentials/udfs/common/string/string_udf.cpp index 92a1fe6b5af..9d2920ec804 100644 --- a/yql/essentials/udfs/common/string/string_udf.cpp +++ b/yql/essentials/udfs/common/string/string_udf.cpp @@ -1,6 +1,7 @@ #include <yql/essentials/public/udf/udf_allocator.h> #include <yql/essentials/public/udf/udf_helpers.h> #include <yql/essentials/public/udf/udf_value_builder.h> +#include <yql/essentials/public/langver/yql_langver.h> #include <library/cpp/deprecated/split/split_iterator.h> #include <library/cpp/html/pcdata/pcdata.h> @@ -85,18 +86,20 @@ namespace { \ END_SIMPLE_ARROW_UDF(T##udfName, T##udfName##KernelExec::Do) -#define STROKA_UDF(udfName, function) \ - SIMPLE_STRICT_UDF(T##udfName, TOptional<char*>(TOptional<char*>)) { \ - EMPTY_RESULT_ON_EMPTY_ARG(0) \ - const TString input(args[0].AsStringRef()); \ - try { \ - TUtf16String wide = UTF8ToWide(input); \ - function(wide); \ - return valueBuilder->NewString(WideToUTF8(wide)); \ - } catch (yexception&) { \ - return TUnboxedValue(); \ - } \ +// NOTE: The functions below are marked as deprecated, so block implementation +// is not required for them +SIMPLE_STRICT_UDF_OPTIONS(TReverse, TOptional<char*>(TOptional<char*>), + builder.SetMaxLangVer(NYql::MakeLangVersion(2025, 1))) { + EMPTY_RESULT_ON_EMPTY_ARG(0) + const TString input(args[0].AsStringRef()); + try { + TUtf16String wide = UTF8ToWide(input); + ReverseInPlace(wide); + return valueBuilder->NewString(WideToUTF8(wide)); + } catch (yexception&) { + return TUnboxedValue(); } +} #define STROKA_CASE_UDF(udfName, function) \ SIMPLE_STRICT_UDF(T##udfName, TOptional<char*>(TOptional<char*>)) { \ @@ -876,7 +879,6 @@ namespace { STRING_UDF_MAP(STRING_UDF) STRING_UNSAFE_UDF_MAP(STRING_UNSAFE_UDF) - STROKA_UDF_MAP(STROKA_UDF) STROKA_CASE_UDF_MAP(STROKA_CASE_UDF) STROKA_ASCII_CASE_UDF_MAP(STROKA_ASCII_CASE_UDF) STROKA_FIND_UDF_MAP(STROKA_FIND_UDF) @@ -902,6 +904,7 @@ namespace { STRING_STREAM_NUM_FORMATTER_UDF_MAP(STRING_REGISTER_UDF) STRING_STREAM_TEXT_FORMATTER_UDF_MAP(STRING_REGISTER_UDF) STRING_STREAM_HRSZ_FORMATTER_UDF_MAP(STRING_REGISTER_UDF) + TReverse, TCollapseText, TReplaceAll, TReplaceFirst, diff --git a/yql/essentials/udfs/common/string/test/canondata/result.json b/yql/essentials/udfs/common/string/test/canondata/result.json index f9e3a670c2c..596f9a72180 100644 --- a/yql/essentials/udfs/common/string/test/canondata/result.json +++ b/yql/essentials/udfs/common/string/test/canondata/result.json @@ -89,6 +89,11 @@ "uri": "file://test.test_Replace_/results.txt" } ], + "test.test[Reverse_2025.02]": [ + { + "uri": "file://test.test_Reverse_2025.02_/extracted" + } + ], "test.test[StreamFormat]": [ { "uri": "file://test.test_StreamFormat_/results.txt" diff --git a/yql/essentials/udfs/common/string/test/canondata/test.test_Reverse_2025.02_/extracted b/yql/essentials/udfs/common/string/test/canondata/test.test_Reverse_2025.02_/extracted new file mode 100644 index 00000000000..2f258b4fc38 --- /dev/null +++ b/yql/essentials/udfs/common/string/test/canondata/test.test_Reverse_2025.02_/extracted @@ -0,0 +1,11 @@ +<tmp_path>/program.sql:<main>: Error: Type annotation + + <tmp_path>/program.sql:<main>:2:1: Error: At function: RemovePrefixMembers, At function: Unordered, At function: PersistableRepr, At function: OrderedSqlProject, At function: SqlProjectItem + select String::Reverse('ab') + ^ + <tmp_path>/program.sql:<main>:2:16: Error: At function: Apply, At function: Udf + select String::Reverse('ab') + ^ + <tmp_path>/program.sql:<main>:2:16: Error: UDF 'String.Reverse' is not available after version 2025.01 + select String::Reverse('ab') + ^
\ No newline at end of file diff --git a/yql/essentials/udfs/common/string/test/cases/Reverse_2025.02.cfg b/yql/essentials/udfs/common/string/test/cases/Reverse_2025.02.cfg new file mode 100644 index 00000000000..989226cf3d4 --- /dev/null +++ b/yql/essentials/udfs/common/string/test/cases/Reverse_2025.02.cfg @@ -0,0 +1,2 @@ +xfail +langver 2025.02 diff --git a/yql/essentials/udfs/common/string/test/cases/Reverse_2025.02.sql b/yql/essentials/udfs/common/string/test/cases/Reverse_2025.02.sql new file mode 100644 index 00000000000..3b388c36154 --- /dev/null +++ b/yql/essentials/udfs/common/string/test/cases/Reverse_2025.02.sql @@ -0,0 +1,2 @@ +select String::Reverse('ab') + diff --git a/yql/essentials/udfs/common/string/ya.make b/yql/essentials/udfs/common/string/ya.make index 280686fb467..bd83c78b8c1 100644 --- a/yql/essentials/udfs/common/string/ya.make +++ b/yql/essentials/udfs/common/string/ya.make @@ -2,7 +2,7 @@ YQL_UDF_CONTRIB(string_udf) YQL_ABI_VERSION( 2 - 37 + 43 0 ) @@ -11,6 +11,7 @@ YQL_UDF_CONTRIB(string_udf) ) PEERDIR( + yql/essentials/public/langver yql/essentials/public/udf/arrow library/cpp/deprecated/split library/cpp/html/pcdata diff --git a/yt/yql/providers/yt/gateway/file/yql_yt_file.cpp b/yt/yql/providers/yt/gateway/file/yql_yt_file.cpp index a3caeb351c2..b292f281fba 100644 --- a/yt/yql/providers/yt/gateway/file/yql_yt_file.cpp +++ b/yt/yql/providers/yt/gateway/file/yql_yt_file.cpp @@ -164,9 +164,10 @@ struct TSession { struct TFileYtLambdaBuilder: public TLambdaBuilder { TFileYtLambdaBuilder(TScopedAlloc& alloc, const TSession& /*session*/, TIntrusivePtr<IFunctionRegistry> customFunctionRegistry, - const NUdf::ISecureParamsProvider* secureParamsProvider) + const NUdf::ISecureParamsProvider* secureParamsProvider, + TLangVersion langver) : TLambdaBuilder(customFunctionRegistry.Get(), alloc, nullptr, CreateDeterministicRandomProvider(1), CreateDeterministicTimeProvider(10000000), - nullptr, nullptr, secureParamsProvider) + nullptr, nullptr, secureParamsProvider, nullptr, langver) , CustomFunctionRegistry_(customFunctionRegistry) {} @@ -445,7 +446,8 @@ public: auto secureParamsProvider = MakeSimpleSecureParamsProvider(options.SecureParams()); TVector<TFileLinkPtr> externalFiles; TFileYtLambdaBuilder builder(alloc, *session, - MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(), Services_->GetFileStorage(), externalFiles), secureParamsProvider.get()); + MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(), + Services_->GetFileStorage(), externalFiles), secureParamsProvider.get(), options.LangVer()); TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), *Services_->GetFunctionRegistry()); TVector<TRuntimeNode> strings; @@ -773,7 +775,8 @@ public: auto secureParamsProvider = MakeSimpleSecureParamsProvider(options.SecureParams()); TVector<TFileLinkPtr> externalFiles; TFileYtLambdaBuilder builder(alloc, *session, - MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(), Services_->GetFileStorage(), externalFiles), secureParamsProvider.get()); + MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(), + Services_->GetFileStorage(), externalFiles), secureParamsProvider.get(), options.LangVer()); auto nodeFactory = GetYtFileFullFactory(Services_); for (auto& node: nodes) { auto data = builder.BuildLambda(*MkqlCompiler_, node, ctx); @@ -819,7 +822,8 @@ public: alloc.SetLimit(options.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); TVector<TFileLinkPtr> externalFiles; TFileYtLambdaBuilder builder(alloc, *session, - MakeFunctionRegistry(*Services_->GetFunctionRegistry(), {}, Services_->GetFileStorage(), externalFiles), nullptr); + MakeFunctionRegistry(*Services_->GetFunctionRegistry(), {}, Services_->GetFileStorage(), externalFiles), + nullptr, UnknownLangVersion); TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), builder.GetFunctionRegistry()); TMkqlBuildContext ctx(*MkqlCompiler_, pgmBuilder, exprCtx); @@ -1252,7 +1256,8 @@ private: auto secureParamsProvider = MakeSimpleSecureParamsProvider(options.SecureParams()); TVector<TFileLinkPtr> externalFiles; TFileYtLambdaBuilder builder(alloc, session, - MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(), Services_->GetFileStorage(), externalFiles), secureParamsProvider.get()); + MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(), + Services_->GetFileStorage(), externalFiles), secureParamsProvider.get(), options.LangVer()); auto data = builder.BuildLambda(*MkqlCompiler_, input.Ptr(), exprCtx); auto transform = TFileTransformProvider(Services_, options.UserDataBlocks()); data = builder.TransformAndOptimizeProgram(data, transform); @@ -1331,7 +1336,8 @@ private: auto secureParamsProvider = MakeSimpleSecureParamsProvider(options.SecureParams()); TVector<TFileLinkPtr> externalFiles; TFileYtLambdaBuilder builder(alloc, session, - MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(), Services_->GetFileStorage(), externalFiles), secureParamsProvider.get()); + MakeFunctionRegistry(*Services_->GetFunctionRegistry(), options.UserDataBlocks(), Services_->GetFileStorage(), + externalFiles), secureParamsProvider.get(), options.LangVer()); auto data = builder.BuildLambda(*MkqlCompiler_, node, exprCtx); auto transform = TFileTransformProvider(Services_, options.UserDataBlocks()); data = builder.TransformAndOptimizeProgram(data, transform); diff --git a/yt/yql/providers/yt/gateway/native/yql_yt_lambda_builder.cpp b/yt/yql/providers/yt/gateway/native/yql_yt_lambda_builder.cpp index cc9981ccf1d..9005ce4170d 100644 --- a/yt/yql/providers/yt/gateway/native/yql_yt_lambda_builder.cpp +++ b/yt/yql/providers/yt/gateway/native/yql_yt_lambda_builder.cpp @@ -100,14 +100,14 @@ NKikimr::NMiniKQL::TComputationNodeFactory GetGatewayNodeFactory(TCodecContext* TNativeYtLambdaBuilder::TNativeYtLambdaBuilder(TScopedAlloc& alloc, const IFunctionRegistry* functionRegistry, const TSession& session, - const NKikimr::NUdf::ISecureParamsProvider* secureParamsProvider) + const NKikimr::NUdf::ISecureParamsProvider* secureParamsProvider, TLangVersion langver) : TLambdaBuilder(functionRegistry, alloc, nullptr, - session.RandomProvider_, session.TimeProvider_, nullptr, nullptr, secureParamsProvider) + session.RandomProvider_, session.TimeProvider_, nullptr, nullptr, secureParamsProvider, nullptr, langver) { } -TNativeYtLambdaBuilder::TNativeYtLambdaBuilder(TScopedAlloc& alloc, const TYtNativeServices& services, const TSession& session) - : TNativeYtLambdaBuilder(alloc, services.FunctionRegistry, session) +TNativeYtLambdaBuilder::TNativeYtLambdaBuilder(TScopedAlloc& alloc, const TYtNativeServices& services, const TSession& session, TLangVersion langver) + : TNativeYtLambdaBuilder(alloc, services.FunctionRegistry, session, nullptr, langver) { } diff --git a/yt/yql/providers/yt/gateway/native/yql_yt_lambda_builder.h b/yt/yql/providers/yt/gateway/native/yql_yt_lambda_builder.h index d147ad4c650..7985054a6e0 100644 --- a/yt/yql/providers/yt/gateway/native/yql_yt_lambda_builder.h +++ b/yt/yql/providers/yt/gateway/native/yql_yt_lambda_builder.h @@ -31,9 +31,11 @@ struct TSession; struct TNativeYtLambdaBuilder: public TLambdaBuilder { TNativeYtLambdaBuilder(NKikimr::NMiniKQL::TScopedAlloc& alloc, const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, - const TSession& session, const NKikimr::NUdf::ISecureParamsProvider* secureParamsProvider = nullptr); + const TSession& session, const NKikimr::NUdf::ISecureParamsProvider* secureParamsProvider, + TLangVersion langver); - TNativeYtLambdaBuilder(NKikimr::NMiniKQL::TScopedAlloc& alloc, const TYtNativeServices& services, const TSession& session); + TNativeYtLambdaBuilder(NKikimr::NMiniKQL::TScopedAlloc& alloc, const TYtNativeServices& services, const TSession& session, + TLangVersion langver); TString BuildLambdaWithIO(const NCommon::IMkqlCallableCompiler& compiler, NNodes::TCoLambda lambda, TExprContext& exprCtx); }; diff --git a/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp b/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp index 2465ceac4cd..460b607e01a 100644 --- a/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp +++ b/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp @@ -1006,7 +1006,7 @@ public: TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), Services_.FunctionRegistry->SupportsSizedAllocators()); alloc.SetLimit(options.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); - TNativeYtLambdaBuilder builder(alloc, Services_, *session); + TNativeYtLambdaBuilder builder(alloc, Services_, *session, options.LangVer()); TVector<TRuntimeNode> tupleNodes; for (auto& node: nodes) { tupleNodes.push_back(builder.BuildLambda(*MkqlCompiler_, node, ctx)); @@ -2314,7 +2314,7 @@ private: TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), execCtx->FunctionRegistry_->SupportsSizedAllocators()); alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); - TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_); + TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_, nullptr, execCtx->Options_.LangVer()); TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), *execCtx->FunctionRegistry_); TRuntimeNode root = DeserializeRuntimeNode(filterLambda, builder.GetTypeEnvironment()); @@ -3493,7 +3493,7 @@ private: TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), Services_.FunctionRegistry->SupportsSizedAllocators()); alloc.SetLimit(options.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); - TNativeYtLambdaBuilder builder(alloc, Services_, *session); + TNativeYtLambdaBuilder builder(alloc, Services_, *session, options.LangVer()); auto rootNode = builder.BuildLambda(*MkqlCompiler_, result.Input().Ptr(), ctx); hasListResult = rootNode.GetStaticType()->IsList(); lambda = SerializeRuntimeNode(rootNode, builder.GetTypeEnvironment()); @@ -3869,7 +3869,7 @@ private: TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), execCtx->FunctionRegistry_->SupportsSizedAllocators()); alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); - TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_); + TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_, nullptr, execCtx->Options_.LangVer()); TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), *execCtx->FunctionRegistry_); TGatewayTransformer transform(execCtx, entry, pgmBuilder, *tmpFiles); size_t nodeCount = 0; @@ -3881,6 +3881,7 @@ private: job->SetOptLLVM(execCtx->Options_.OptLLVM()); job->SetUdfValidateMode(execCtx->Options_.UdfValidateMode()); job->SetRuntimeLogLevel(execCtx->Options_.RuntimeLogLevel()); + job->SetLangVer(execCtx->Options_.LangVer()); transform.ApplyJobProps(*job); transform.ApplyUserJobSpec(userJobSpec, testRun); @@ -3956,7 +3957,7 @@ private: TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), execCtx->FunctionRegistry_->SupportsSizedAllocators()); alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); - TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_); + TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_, execCtx->Options_.LangVer()); mapLambda = builder.BuildLambdaWithIO(*MkqlCompiler_, map.Mapper(), ctx); } @@ -4093,7 +4094,7 @@ private: TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), execCtx->FunctionRegistry_->SupportsSizedAllocators()); alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); - TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_); + TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_, nullptr, execCtx->Options_.LangVer()); TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), *execCtx->FunctionRegistry_); TGatewayTransformer transform(execCtx, entry, pgmBuilder, *tmpFiles); size_t nodeCount = 0; @@ -4105,6 +4106,7 @@ private: job->SetOptLLVM(execCtx->Options_.OptLLVM()); job->SetUdfValidateMode(execCtx->Options_.UdfValidateMode()); job->SetRuntimeLogLevel(execCtx->Options_.RuntimeLogLevel()); + job->SetLangVer(execCtx->Options_.LangVer()); transform.ApplyJobProps(*job); transform.ApplyUserJobSpec(userJobSpec, testRun); FillUserJobSpec(userJobSpec, execCtx, extraUsage, transform.GetUsedMemory(), execCtx->EstimateLLVMMem(nodeCount), testRun); @@ -4163,7 +4165,7 @@ private: TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), execCtx->FunctionRegistry_->SupportsSizedAllocators()); alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); - TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_); + TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_, execCtx->Options_.LangVer()); reduceLambda = builder.BuildLambdaWithIO(*MkqlCompiler_, reduce.Reducer(), ctx); } @@ -4347,7 +4349,7 @@ private: TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), execCtx->FunctionRegistry_->SupportsSizedAllocators()); alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); - TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_); + TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_, nullptr, execCtx->Options_.LangVer()); TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), *execCtx->FunctionRegistry_); TGatewayTransformer transform(execCtx, entry, pgmBuilder, *tmpFiles); size_t nodeCount = 0; @@ -4359,6 +4361,7 @@ private: mapJob->SetOptLLVM(execCtx->Options_.OptLLVM()); mapJob->SetUdfValidateMode(execCtx->Options_.UdfValidateMode()); mapJob->SetRuntimeLogLevel(execCtx->Options_.RuntimeLogLevel()); + mapJob->SetLangVer(execCtx->Options_.LangVer()); transform.ApplyJobProps(*mapJob); transform.ApplyUserJobSpec(mapUserJobSpec, testRun); @@ -4377,7 +4380,7 @@ private: TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), execCtx->FunctionRegistry_->SupportsSizedAllocators()); alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); - TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_); + TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_, nullptr, execCtx->Options_.LangVer()); TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), *execCtx->FunctionRegistry_); TGatewayTransformer transform(execCtx, entry, pgmBuilder, *tmpFiles); size_t nodeCount = 0; @@ -4389,6 +4392,7 @@ private: reduceJob->SetOptLLVM(execCtx->Options_.OptLLVM()); reduceJob->SetUdfValidateMode(execCtx->Options_.UdfValidateMode()); reduceJob->SetRuntimeLogLevel(execCtx->Options_.RuntimeLogLevel()); + reduceJob->SetLangVer(execCtx->Options_.LangVer()); transform.ApplyJobProps(*reduceJob); transform.ApplyUserJobSpec(reduceUserJobSpec, testRun); FillUserJobSpec(reduceUserJobSpec, execCtx, reduceExtraUsage, transform.GetUsedMemory(), execCtx->EstimateLLVMMem(nodeCount), testRun); @@ -4532,7 +4536,7 @@ private: TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), execCtx->FunctionRegistry_->SupportsSizedAllocators()); alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); - TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_); + TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_, nullptr, execCtx->Options_.LangVer()); TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), *execCtx->FunctionRegistry_); TGatewayTransformer transform(execCtx, entry, pgmBuilder, *tmpFiles); size_t nodeCount = 0; @@ -4544,6 +4548,7 @@ private: reduceJob->SetOptLLVM(execCtx->Options_.OptLLVM()); reduceJob->SetUdfValidateMode(execCtx->Options_.UdfValidateMode()); reduceJob->SetRuntimeLogLevel(execCtx->Options_.RuntimeLogLevel()); + reduceJob->SetLangVer(execCtx->Options_.LangVer()); transform.ApplyJobProps(*reduceJob); transform.ApplyUserJobSpec(reduceUserJobSpec, testRun); FillUserJobSpec(reduceUserJobSpec, execCtx, reduceExtraUsage, transform.GetUsedMemory(), execCtx->EstimateLLVMMem(nodeCount), testRun); @@ -4669,7 +4674,7 @@ private: TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), execCtx->FunctionRegistry_->SupportsSizedAllocators()); alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); - TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_); + TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_, execCtx->Options_.LangVer()); mapLambda = builder.BuildLambdaWithIO(*MkqlCompiler_, mapReduce.Mapper().Cast<TCoLambda>(), ctx); mapInputType = NCommon::WriteTypeToYson(GetSequenceItemType(mapReduce.Input().Size() == 1U ? TExprBase(mapReduce.Input().Item(0)) : TExprBase(mapReduce.Mapper().Cast<TCoLambda>().Args().Arg(0)), true)); @@ -4688,7 +4693,7 @@ private: TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), execCtx->FunctionRegistry_->SupportsSizedAllocators()); alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); - TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_); + TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_, execCtx->Options_.LangVer()); reduceLambda = builder.BuildLambdaWithIO(*MkqlCompiler_, mapReduce.Reducer(), ctx); } TExpressionResorceUsage reduceExtraUsage = execCtx->ScanExtraResourceUsage(mapReduce.Reducer().Body().Ref(), false); @@ -4831,7 +4836,7 @@ private: TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), execCtx->FunctionRegistry_->SupportsSizedAllocators()); alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); - TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_); + TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_, nullptr, execCtx->Options_.LangVer()); TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), *execCtx->FunctionRegistry_); TGatewayTransformer transform(execCtx, entry, pgmBuilder, *tmpFiles); transform.SetTwoPhaseTransform(); @@ -4872,6 +4877,7 @@ private: job->SetOptLLVM(execCtx->Options_.OptLLVM()); job->SetUdfValidateMode(execCtx->Options_.UdfValidateMode()); job->SetRuntimeLogLevel(execCtx->Options_.RuntimeLogLevel()); + job->SetLangVer(execCtx->Options_.LangVer()); const auto nativeTypeCompat = execCtx->Options_.Config()->NativeYtTypeCompatibility.Get(execCtx->Cluster_).GetOrElse(NTCF_LEGACY); job->SetOutSpec(execCtx->GetOutSpec(!useSkiff, nativeTypeCompat)); job->SetUseSkiff(useSkiff, 0); @@ -4935,7 +4941,7 @@ private: TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), Services_.FunctionRegistry->SupportsSizedAllocators()); alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); - TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_); + TNativeYtLambdaBuilder builder(alloc, Services_, *execCtx->Session_, execCtx->Options_.LangVer()); lambda = builder.BuildLambdaWithIO(*MkqlCompiler_, fill.Content(), ctx); } auto extraUsage = execCtx->ScanExtraResourceUsage(fill.Content().Ref(), false); @@ -5432,7 +5438,7 @@ private: execCtx->FunctionRegistry_->SupportsSizedAllocators()); alloc.SetLimit(execCtx->Options_.Config()->DefaultCalcMemoryLimit.Get().GetOrElse(0)); auto secureParamsProvider = MakeSimpleSecureParamsProvider(execCtx->Options_.SecureParams()); - TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_, secureParamsProvider.get()); + TNativeYtLambdaBuilder builder(alloc, execCtx->FunctionRegistry_, *execCtx->Session_, secureParamsProvider.get(), execCtx->Options_.LangVer()); THolder<TCodecContext> codecCtx; TString pathPrefix; TProgramBuilder pgmBuilder(builder.GetTypeEnvironment(), *execCtx->FunctionRegistry_); @@ -5507,6 +5513,7 @@ private: job->SetOptLLVM(execCtx->Options_.OptLLVM()); job->SetUdfValidateMode(execCtx->Options_.UdfValidateMode()); job->SetRuntimeLogLevel(execCtx->Options_.RuntimeLogLevel()); + job->SetLangVer(execCtx->Options_.LangVer()); mapOpSpec.AddInput(tmpTable); mapOpSpec.AddOutput(tmpTable); diff --git a/yt/yql/providers/yt/job/ya.make b/yt/yql/providers/yt/job/ya.make index fe892d4f71f..f67ac0d8c2a 100644 --- a/yt/yql/providers/yt/job/ya.make +++ b/yt/yql/providers/yt/job/ya.make @@ -18,6 +18,7 @@ PEERDIR( yt/cpp/mapreduce/interface yt/cpp/mapreduce/library/user_job_statistics yql/essentials/minikql/comp_nodes + yql/essentials/public/langver yql/essentials/public/udf yql/essentials/utils yql/essentials/utils/backtrace diff --git a/yt/yql/providers/yt/job/yql_job_base.cpp b/yt/yql/providers/yt/job/yql_job_base.cpp index 00d96a9ca2e..0874eb45244 100644 --- a/yt/yql/providers/yt/job/yql_job_base.cpp +++ b/yt/yql/providers/yt/job/yql_job_base.cpp @@ -286,7 +286,8 @@ void TYqlJobBase::Save(IOutputStream& s) const { UdfValidateMode, OptLLVM, TableNames, - RuntimeLogLevel + RuntimeLogLevel, + LangVer ); } @@ -297,7 +298,8 @@ void TYqlJobBase::Load(IInputStream& s) { UdfValidateMode, OptLLVM, TableNames, - RuntimeLogLevel + RuntimeLogLevel, + LangVer ); } diff --git a/yt/yql/providers/yt/job/yql_job_base.h b/yt/yql/providers/yt/job/yql_job_base.h index e9bf6b7c66e..eb9e02587dc 100644 --- a/yt/yql/providers/yt/job/yql_job_base.h +++ b/yt/yql/providers/yt/job/yql_job_base.h @@ -1,6 +1,7 @@ #pragma once #include <yql/essentials/providers/common/codec/yql_codec.h> +#include <yql/essentials/public/langver/yql_langver.h> #include <yql/essentials/public/udf/udf_validate.h> #include <yql/essentials/public/udf/udf_counter.h> #include <yql/essentials/minikql/mkql_node_visitor.h> @@ -74,6 +75,10 @@ public: RuntimeLogLevel = level; } + void SetLangVer(TLangVersion langver) { + LangVer = langver; + } + void Do(const NYT::TRawJobContext& jobContext) override; void Save(IOutputStream& stream) const override; void Load(IInputStream& stream) override; @@ -93,6 +98,7 @@ protected: TString OptLLVM; TVector<TString> TableNames; NUdf::ELogLevel RuntimeLogLevel = NUdf::ELogLevel::Info; + TLangVersion LangVer = UnknownLangVersion; // End serializable part ui64 StartCycles = 0; diff --git a/yt/yql/providers/yt/job/yql_job_user.cpp b/yt/yql/providers/yt/job/yql_job_user.cpp index 0b355e6609d..edbcce66509 100644 --- a/yt/yql/providers/yt/job/yql_job_user.cpp +++ b/yt/yql/providers/yt/job/yql_job_user.cpp @@ -150,7 +150,7 @@ void TYqlUserJob::DoImpl(const TFile& inHandle, const TVector<TFile>& outHandles TLambdaBuilder builder(FunctionRegistry.Get(), *Alloc, Env.Get(), RandomProvider.Get(), TimeProvider.Get(), JobStats.Get(), &JobCountersProvider, - SecureParamsProvider.Get(), LogProvider.Get()); + SecureParamsProvider.Get(), LogProvider.Get(), LangVer); TType* itemType = nullptr; if (InputType) { diff --git a/yt/yql/providers/yt/lib/lambda_builder/lambda_builder.cpp b/yt/yql/providers/yt/lib/lambda_builder/lambda_builder.cpp index d706a71a7a3..cfca9dcbb8a 100644 --- a/yt/yql/providers/yt/lib/lambda_builder/lambda_builder.cpp +++ b/yt/yql/providers/yt/lib/lambda_builder/lambda_builder.cpp @@ -23,7 +23,8 @@ TLambdaBuilder::TLambdaBuilder(const NKikimr::NMiniKQL::IFunctionRegistry* funct NKikimr::NMiniKQL::IStatsRegistry* jobStats, NKikimr::NUdf::ICountersProvider* counters, const NKikimr::NUdf::ISecureParamsProvider* secureParamsProvider, - const NKikimr::NUdf::ILogProvider* logProvider) + const NKikimr::NUdf::ILogProvider* logProvider, + TLangVersion langVer) : FunctionRegistry(functionRegistry) , Alloc(alloc) , RandomProvider(randomProvider) @@ -32,6 +33,7 @@ TLambdaBuilder::TLambdaBuilder(const NKikimr::NMiniKQL::IFunctionRegistry* funct , Counters(counters) , SecureParamsProvider(secureParamsProvider) , LogProvider(logProvider) + , LangVer(langVer) , Env(env) { } @@ -52,7 +54,7 @@ const NKikimr::NMiniKQL::TTypeEnvironment* TLambdaBuilder::CreateTypeEnv() const TRuntimeNode TLambdaBuilder::BuildLambda(const IMkqlCallableCompiler& compiler, const TExprNode::TPtr& lambdaNode, TExprContext& exprCtx, TArgumentsMap&& arguments) const { - TProgramBuilder pgmBuilder(GetTypeEnvironment(), *FunctionRegistry); + TProgramBuilder pgmBuilder(GetTypeEnvironment(), *FunctionRegistry, false, LangVer); TMkqlBuildContext ctx(compiler, pgmBuilder, exprCtx, lambdaNode->UniqueId(), std::move(arguments)); return MkqlBuildExpr(*lambdaNode, ctx); } @@ -184,7 +186,7 @@ THolder<IComputationGraph> TLambdaBuilder::BuildGraph( TComputationPatternOpts patternOpts(Alloc.Ref(), GetTypeEnvironment()); patternOpts.SetOptions(factory, FunctionRegistry, validateMode, validatePolicy, optLLVM, graphPerProcess, JobStats, Counters, - SecureParamsProvider, LogProvider); + SecureParamsProvider, LogProvider, LangVer); auto preparePatternFunc = [&]() { if (serialized) { auto tupleRunTimeNodes = DeserializeRuntimeNode(serialized, GetTypeEnvironment()); @@ -206,13 +208,13 @@ THolder<IComputationGraph> TLambdaBuilder::BuildGraph( YQL_ENSURE(pattern); const TComputationOptsFull computeOpts(JobStats, Alloc.Ref(), GetTypeEnvironment(), *randomProvider, *timeProvider, - validatePolicy, SecureParamsProvider, Counters, LogProvider); + validatePolicy, SecureParamsProvider, Counters, LogProvider, LangVer); auto graph = pattern->Clone(computeOpts); return MakeHolder<TComputationGraphProxy>(std::move(pattern), std::move(graph)); } TRuntimeNode TLambdaBuilder::MakeTuple(const TVector<TRuntimeNode>& items) const { - TProgramBuilder pgmBuilder(GetTypeEnvironment(), *FunctionRegistry); + TProgramBuilder pgmBuilder(GetTypeEnvironment(), *FunctionRegistry, false, LangVer); return pgmBuilder.NewTuple(items); } diff --git a/yt/yql/providers/yt/lib/lambda_builder/lambda_builder.h b/yt/yql/providers/yt/lib/lambda_builder/lambda_builder.h index c1f7aca1b41..20f3226a2f8 100644 --- a/yt/yql/providers/yt/lib/lambda_builder/lambda_builder.h +++ b/yt/yql/providers/yt/lib/lambda_builder/lambda_builder.h @@ -10,6 +10,7 @@ #include <yql/essentials/minikql/mkql_node.h> #include <yql/essentials/minikql/mkql_alloc.h> #include <yql/essentials/public/udf/udf_validate.h> +#include <yql/essentials/public/langver/yql_langver.h> #include <library/cpp/random_provider/random_provider.h> #include <library/cpp/time_provider/time_provider.h> @@ -35,7 +36,8 @@ public: NKikimr::NMiniKQL::IStatsRegistry* jobStats = nullptr, NKikimr::NUdf::ICountersProvider* counters = nullptr, const NKikimr::NUdf::ISecureParamsProvider *secureParamsProvider = nullptr, - const NKikimr::NUdf::ILogProvider* logProvider = nullptr); + const NKikimr::NUdf::ILogProvider* logProvider = nullptr, + TLangVersion langver = UnknownLangVersion); ~TLambdaBuilder(); @@ -108,6 +110,7 @@ protected: NKikimr::NUdf::ICountersProvider* const Counters; const NKikimr::NUdf::ISecureParamsProvider* SecureParamsProvider; const NKikimr::NUdf::ILogProvider* LogProvider; + const TLangVersion LangVer; /// TODO: remove? void SetExternalEnv(const NKikimr::NMiniKQL::TTypeEnvironment* env); diff --git a/yt/yql/providers/yt/lib/lambda_builder/ya.make b/yt/yql/providers/yt/lib/lambda_builder/ya.make index 306049a0eee..a0f73ed76be 100644 --- a/yt/yql/providers/yt/lib/lambda_builder/ya.make +++ b/yt/yql/providers/yt/lib/lambda_builder/ya.make @@ -12,6 +12,7 @@ PEERDIR( yql/essentials/public/udf yql/essentials/utils yql/essentials/providers/common/mkql + yql/essentials/public/langver ) YQL_LAST_ABI_VERSION() diff --git a/yt/yql/providers/yt/provider/ya.make b/yt/yql/providers/yt/provider/ya.make index 74cbe7ac426..e705a3476a2 100644 --- a/yt/yql/providers/yt/provider/ya.make +++ b/yt/yql/providers/yt/provider/ya.make @@ -80,6 +80,7 @@ PEERDIR( yt/cpp/mapreduce/interface yql/essentials/ast yql/essentials/core/extract_predicate + yql/essentials/public/langver yql/essentials/public/udf yql/essentials/public/udf/tz yql/essentials/sql diff --git a/yt/yql/providers/yt/provider/yql_yt_datasink_exec.cpp b/yt/yql/providers/yt/provider/yql_yt_datasink_exec.cpp index 05e0ca5caa0..74905bb7b32 100644 --- a/yt/yql/providers/yt/provider/yql_yt_datasink_exec.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_datasink_exec.cpp @@ -314,6 +314,7 @@ private: .OperationHash(operationHash) .SecureParams(secureParams) .RuntimeLogLevel(State_->Types->RuntimeLogLevel) + .LangVer(State_->Types->LangVer) .AdditionalSecurityTags(addSecTags) ); } diff --git a/yt/yql/providers/yt/provider/yql_yt_datasource_exec.cpp b/yt/yql/providers/yt/provider/yql_yt_datasource_exec.cpp index c4ed2d2a34d..971b66209ed 100644 --- a/yt/yql/providers/yt/provider/yql_yt_datasource_exec.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_datasource_exec.cpp @@ -200,6 +200,7 @@ protected: .OperationHash(operationHash) .SecureParams(secureParams) .RuntimeLogLevel(State_->Types->RuntimeLogLevel) + .LangVer(State_->Types->LangVer) ); return WrapFuture(future, [](const IYtGateway::TResOrPullResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { diff --git a/yt/yql/providers/yt/provider/yql_yt_gateway.h b/yt/yql/providers/yt/provider/yql_yt_gateway.h index e1b49abf7b6..a40f1f863f9 100644 --- a/yt/yql/providers/yt/provider/yql_yt_gateway.h +++ b/yt/yql/providers/yt/provider/yql_yt_gateway.h @@ -13,6 +13,7 @@ #include <yql/essentials/core/yql_type_annotation.h> #include <yql/essentials/core/yql_execution.h> #include <yql/essentials/core/file_storage/storage.h> +#include <yql/essentials/public/langver/yql_langver.h> #include <yt/cpp/mapreduce/interface/common.h> @@ -251,6 +252,7 @@ public: OPTION_FIELD(TPosition, Pos) OPTION_FIELD(TSecureParams, SecureParams) OPTION_FIELD_DEFAULT(NUdf::ELogLevel, RuntimeLogLevel, NUdf::ELogLevel::Info) + OPTION_FIELD_DEFAULT(TLangVersion, LangVer, UnknownLangVersion) }; struct TTableRangeResult : public NCommon::TOperationResult { @@ -360,6 +362,7 @@ public: OPTION_FIELD(TString, OperationHash) OPTION_FIELD(TSecureParams, SecureParams) OPTION_FIELD_DEFAULT(NUdf::ELogLevel, RuntimeLogLevel, NUdf::ELogLevel::Info) + OPTION_FIELD_DEFAULT(TLangVersion, LangVer, UnknownLangVersion) }; struct TResOrPullResult : public NCommon::TOperationResult { @@ -386,6 +389,7 @@ public: OPTION_FIELD(TString, OperationHash) OPTION_FIELD(TSecureParams, SecureParams) OPTION_FIELD_DEFAULT(NUdf::ELogLevel, RuntimeLogLevel, NUdf::ELogLevel::Info) + OPTION_FIELD_DEFAULT(TLangVersion, LangVer, UnknownLangVersion) OPTION_FIELD_DEFAULT(TSet<TString>, AdditionalSecurityTags, {}) }; @@ -432,6 +436,7 @@ public: OPTION_FIELD(TString, OperationHash) OPTION_FIELD(TSecureParams, SecureParams) OPTION_FIELD_DEFAULT(NUdf::ELogLevel, RuntimeLogLevel, NUdf::ELogLevel::Info) + OPTION_FIELD_DEFAULT(TLangVersion, LangVer, UnknownLangVersion) }; struct TCalcResult : public NCommon::TOperationResult { diff --git a/yt/yql/providers/yt/provider/yql_yt_helpers.cpp b/yt/yql/providers/yt/provider/yql_yt_helpers.cpp index 887b57d2044..4ee06a21708 100644 --- a/yt/yql/providers/yt/provider/yql_yt_helpers.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_helpers.cpp @@ -950,6 +950,7 @@ std::pair<IGraphTransformer::TStatus, TAsyncTransformCallbackFuture> CalculateNo .OperationHash(calcHash) .SecureParams(secureParams) .RuntimeLogLevel(state->Types->RuntimeLogLevel) + .LangVer(state->Types->LangVer) ); return WrapFutureCallback(future, [state, calcNodes](const IYtGateway::TCalcResult& res, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { YQL_ENSURE(res.Data.size() == calcNodes.size()); diff --git a/yt/yql/providers/yt/provider/yql_yt_io_discovery.cpp b/yt/yql/providers/yt/provider/yql_yt_io_discovery.cpp index feb5f75cb74..fd31f57950c 100644 --- a/yt/yql/providers/yt/provider/yql_yt_io_discovery.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_io_discovery.cpp @@ -307,6 +307,7 @@ public: .Config(State_->Configuration->Snapshot()) .OptLLVM(State_->Types->OptLLVM.GetOrElse(TString())) .RuntimeLogLevel(State_->Types->RuntimeLogLevel) + .LangVer(State_->Types->LangVer) .Pos(x.second.first) ); allFutures.push_back(result.IgnoreResult()); |