YQL-19530 pass cpu & mem settings to Udf/ScriptUdf

commit_hash:21577e2038f1d80c16e671d74e0c9e1e00b6c448

18 files changed, 347 insertions, 46 deletions

diff --git a/yql/essentials/core/type_ann/type_ann_core.cpp b/yql/essentials/core/type_ann/type_ann_core.cpp
index 879f2cbcc64..d56c5bc7604 100644
--- a/yql/essentials/core/type_ann/type_ann_core.cpp
+++ b/yql/essentials/core/type_ann/type_ann_core.cpp
@@ -7532,7 +7532,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
 
         // (4) cached callable type
         const TCallableExprType* cachedType = nullptr;
-        if (input->ChildrenSize() > 4) {
+        if (input->ChildrenSize() > 4 && !input->Child(4)->IsCallable("Void")) {
             if (auto status = EnsureTypeRewrite(input->ChildRef(4), ctx.Expr); status != IGraphTransformer::TStatus::Ok) {
                 return status;
             }
@@ -7566,12 +7566,14 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
         }
 
         // (7) settings
+        TExprNode::TPtr settings;
         if (input->ChildrenSize() > 7) {
-            if (!EnsureTuple(*input->Child(7), ctx.Expr)) {
+            settings = input->ChildPtr(7);
+            if (!EnsureTuple(*settings, ctx.Expr)) {
                 return IGraphTransformer::TStatus::Error;
             }
 
-            for (const auto& child : input->Child(7)->Children()) {
+            for (const auto& child : settings->Children()) {
                 if (!EnsureTupleMinSize(*child, 1, ctx.Expr)) {
                     return IGraphTransformer::TStatus::Error;
                 }
@@ -7589,6 +7591,14 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
                     if (!EnsureTupleSize(*child, 1, ctx.Expr)) {
                         return IGraphTransformer::TStatus::Error;
                     }
+                } else if (settingName == "cpu" || settingName == "extraMem") {
+                    if (!EnsureTupleSize(*child, 2, ctx.Expr)) {
+                        return IGraphTransformer::TStatus::Error;
+                    }
+
+                    if (!EnsureAtom(child->Tail(), ctx.Expr)) {
+                        return IGraphTransformer::TStatus::Error;
+                    }
                 } else {
                     ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Head().Pos()), TStringBuilder() << "Unknown setting: " << settingName));
                     return IGraphTransformer::TStatus::Error;
@@ -7596,7 +7606,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
             }
         }
 
-        if (input->ChildrenSize() != 8) {
+        if (input->ChildrenSize() != 8 || !cachedType) {
             YQL_PROFILE_SCOPE(DEBUG, "ResolveUdfs");
             auto& cached = ctx.Types.UdfTypeCache[std::make_tuple(TString(name), TString(typeConfig), userType)];
             if (!cached.FunctionType) {
@@ -7700,6 +7710,16 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
                                     .Seal();
                             }
 
+                            if (settings) {
+                                if (auto setting = GetSetting(*settings, "cpu")) {
+                                    parent.Add(settingIndex++, setting);
+                                }
+
+                                if (auto setting = GetSetting(*settings, "extraMem")) {
+                                    parent.Add(settingIndex++, setting);
+                                }
+                            }
+
                             return parent;
                         })
                     .Seal()
@@ -8271,7 +8291,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
     }
 
     IGraphTransformer::TStatus SqlCallWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
-        if (!EnsureMinMaxArgsCount(*input, 2, 5, ctx.Expr)) {
+        if (!EnsureMinMaxArgsCount(*input, 2, 6, ctx.Expr)) {
             return IGraphTransformer::TStatus::Error;
         }
 
@@ -8331,6 +8351,11 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
             }
         }
 
+        TExprNode::TPtr options;
+        if (input->ChildrenSize() > 5) {
+            options = input->Child(5);
+        }
+
         TExprNode::TPtr udf = ctx.Expr.Builder(input->Pos())
             .Callable("Udf")
                 .Add(0, udfName)
@@ -8377,7 +8402,17 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
                 .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
                     if (typeConfig) {
                         parent.Add(3, typeConfig);
+                    } else if (options) {
+                        parent.Atom(3, "");
+                    }
+
+                    if (options) {
+                        parent.Callable(4,"Void").Seal();
+                        parent.Callable(5,"VoidType").Seal();
+                        parent.Atom(6,"");
+                        parent.Add(7, options);
                     }
+
                     return parent;
                 })
             .Seal()
diff --git a/yql/essentials/sql/v1/builtin.cpp b/yql/essentials/sql/v1/builtin.cpp
index 8e49ba39f22..2cd2aeaee06 100644
--- a/yql/essentials/sql/v1/builtin.cpp
+++ b/yql/essentials/sql/v1/builtin.cpp
@@ -2286,7 +2286,8 @@ TVector<TNodePtr> BuildUdfArgs(const TContext& ctx, TPosition pos, const TVector
 }
 
 TNodePtr BuildSqlCall(TContext& ctx, TPosition pos, const TString& module, const TString& name, const TVector<TNodePtr>& args,
-    TNodePtr positionalArgs, TNodePtr namedArgs, TNodePtr customUserType, const TDeferredAtom& typeConfig, TNodePtr runConfig)
+    TNodePtr positionalArgs, TNodePtr namedArgs, TNodePtr customUserType, const TDeferredAtom& typeConfig, TNodePtr runConfig,
+    TNodePtr options)
 {
     const TString fullName = module + "." + name;
     TNodePtr callable;
@@ -2318,18 +2319,24 @@ TNodePtr BuildSqlCall(TContext& ctx, TPosition pos, const TString& module, const
     // optional arguments
     if (customUserType) {
         sqlCallArgs.push_back(customUserType);
-    } else if (!typeConfig.Empty()) {
+    } else if (!typeConfig.Empty() || runConfig || options) {
         sqlCallArgs.push_back(new TCallNodeImpl(pos, "TupleType", {}));
     }
 
     if (!typeConfig.Empty()) {
         sqlCallArgs.push_back(typeConfig.Build());
-    } else if (runConfig) {
+    } else if (runConfig || options) {
         sqlCallArgs.push_back(BuildQuotedAtom(pos, ""));
     }
 
     if (runConfig) {
         sqlCallArgs.push_back(runConfig);
+    } else if (options) {
+        sqlCallArgs.push_back(new TCallNodeImpl(pos, "Void", {}));
+    }
+
+    if (options) {
+        sqlCallArgs.push_back(options);
     }
 
     return new TCallNodeImpl(pos, "SqlCall", sqlCallArgs);
@@ -2463,45 +2470,47 @@ TNodePtr BuildUdf(TContext& ctx, TPosition pos, const TString& module, const TSt
 
 class TScriptUdf final: public INode {
 public:
-    TScriptUdf(TPosition pos, const TString& moduleName, const TString& funcName, const TVector<TNodePtr>& args)
+    TScriptUdf(TPosition pos, const TString& moduleName, const TString& funcName, const TVector<TNodePtr>& args,
+        TNodePtr options)
         : INode(pos)
-        , ModuleName(moduleName)
-        , FuncName(funcName)
-        , Args(args)
+        , ModuleName_(moduleName)
+        , FuncName_(funcName)
+        , Args_(args)
+        , Options_(options)
     {}
 
     bool DoInit(TContext& ctx, ISource* src) override {
-        const bool isPython = ModuleName.find(TStringBuf("Python")) != TString::npos;
+        const bool isPython = ModuleName_.find(TStringBuf("Python")) != TString::npos;
         if (!isPython) {
-            if (Args.size() != 2) {
-                ctx.Error(Pos) << ModuleName << " script declaration requires exactly two parameters";
+            if (Args_.size() != 2) {
+                ctx.Error(Pos) << ModuleName_ << " script declaration requires exactly two parameters";
                 return false;
             }
         } else {
-            if (Args.size() < 1 || Args.size() > 2) {
-                ctx.Error(Pos) << ModuleName << " script declaration requires one or two parameters";
+            if (Args_.size() < 1 || Args_.size() > 2) {
+                ctx.Error(Pos) << ModuleName_ << " script declaration requires one or two parameters";
                 return false;
             }
         }
 
-        auto nameAtom = BuildQuotedAtom(Pos, FuncName);
-        auto scriptNode = Args.back();
+        auto nameAtom = BuildQuotedAtom(Pos, FuncName_);
+        auto scriptNode = Args_.back();
         if (!scriptNode->Init(ctx, src)) {
             return false;
         }
-        auto scriptStrPtr = Args.back()->GetLiteral("String");
+        auto scriptStrPtr = Args_.back()->GetLiteral("String");
         if (!ctx.CompactNamedExprs && scriptStrPtr && scriptStrPtr->size() > SQL_MAX_INLINE_SCRIPT_LEN) {
             scriptNode = ctx.UniversalAlias("scriptudf", std::move(scriptNode));
         }
 
         INode::TPtr type;
-        if (Args.size() == 2) {
-            type = Args[0];
+        if (Args_.size() == 2) {
+            type = Args_[0];
         } else {
             // Python supports getting functions signatures right from docstrings
             type = Y("EvaluateType", Y("ParseTypeHandle", Y("Apply",
                 Y("bind", "core_module", Q("PythonFuncSignature")),
-                Q(ModuleName),
+                Q(ModuleName_),
                 scriptNode,
                 Y("String", nameAtom)
                 )));
@@ -2511,14 +2520,18 @@ public:
             return false;
         }
 
-        Node = Y("ScriptUdf", Q(ModuleName), nameAtom, type, scriptNode);
+        Node_ = Y("ScriptUdf", Q(ModuleName_), nameAtom, type, scriptNode);
+        if (Options_) {
+            Node_ = L(Node_, Options_);
+        }
+
         return true;
     }
 
     TAstNode* Translate(TContext& ctx) const override {
         Y_UNUSED(ctx);
-        Y_DEBUG_ABORT_UNLESS(Node);
-        return Node->Translate(ctx);
+        Y_DEBUG_ABORT_UNLESS(Node_);
+        return Node_->Translate(ctx);
     }
 
     void DoUpdateState() const override {
@@ -2526,20 +2539,47 @@ public:
     }
 
     TNodePtr DoClone() const final {
-        return new TScriptUdf(GetPos(), ModuleName, FuncName, CloneContainer(Args));
+        return new TScriptUdf(GetPos(), ModuleName_, FuncName_, CloneContainer(Args_), Options_);
     }
 
     void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const final {
-        Y_DEBUG_ABORT_UNLESS(Node);
-        Node->VisitTree(func, visited);
+        Y_DEBUG_ABORT_UNLESS(Node_);
+        Node_->VisitTree(func, visited);
+    }
+
+    const TString* FuncName() const final {
+        return &FuncName_;
+    }
+
+    const TString* ModuleName() const final {
+        return &ModuleName_;
+    }
+
+    bool IsScript() const final {
+        return true;
     }
+
+    size_t GetTupleSize() const final {
+        return Args_.size();
+    }
+
+    TPtr GetTupleElement(size_t index) const final {
+        return Args_[index];
+    }
+
 private:
-    TString ModuleName;
-    TString FuncName;
-    TVector<TNodePtr> Args;
-    TNodePtr Node;
+    TString ModuleName_;
+    TString FuncName_;
+    TVector<TNodePtr> Args_;
+    TNodePtr Node_;
+    TNodePtr Options_;
 };
 
+TNodePtr BuildScriptUdf(TPosition pos, const TString& moduleName, const TString& funcName, const TVector<TNodePtr>& args,
+        TNodePtr options) {
+    return new TScriptUdf(pos, moduleName, funcName, args, options);
+}
+
 template <bool Sorted, bool Hashed>
 class TYqlToDict final: public TCallNode {
 public:
@@ -3549,7 +3589,7 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec
         return BuildUdf(ctx, pos, nameSpace, name, makeUdfArgs());
     } else if (scriptType != NKikimr::NMiniKQL::EScriptType::Unknown) {
         auto scriptName = NKikimr::NMiniKQL::IsCustomPython(scriptType) ? nameSpace : TString(NKikimr::NMiniKQL::ScriptTypeAsStr(scriptType));
-        return new TScriptUdf(pos, scriptName, name, args);
+        return BuildScriptUdf(pos, scriptName, name, args, nullptr);
     } else if (ns.empty()) {
         if (auto simpleType = LookupSimpleType(normalizedName, ctx.FlexibleTypes, /* isPgType = */ false)) {
             const auto type = *simpleType;
@@ -3871,7 +3911,8 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec
     }
 
     TNodePtr typeConfig = MakeTypeConfig(pos, ns, usedArgs);
-    return BuildSqlCall(ctx, pos, nameSpace, name, usedArgs, positionalArgs, namedArgs, customUserType, TDeferredAtom(typeConfig, ctx), nullptr);
+    return BuildSqlCall(ctx, pos, nameSpace, name, usedArgs, positionalArgs, namedArgs, customUserType,
+        TDeferredAtom(typeConfig, ctx), nullptr, nullptr);
 }
 
 } // namespace NSQLTranslationV1
diff --git a/yql/essentials/sql/v1/node.cpp b/yql/essentials/sql/v1/node.cpp
index eea9069abf3..e5c7d353795 100644
--- a/yql/essentials/sql/v1/node.cpp
+++ b/yql/essentials/sql/v1/node.cpp
@@ -334,6 +334,10 @@ const TString* INode::ModuleName() const {
     return nullptr;
 }
 
+bool INode::IsScript() const {
+    return false;
+}
+
 bool INode::HasSkip() const {
     return false;
 }
@@ -566,6 +570,10 @@ const TString* IProxyNode::ModuleName() const {
     return Inner->ModuleName();
 }
 
+bool IProxyNode::IsScript() const {
+    return Inner->IsScript();
+}
+
 bool IProxyNode::HasSkip() const {
     return Inner->HasSkip();
 }
@@ -3027,6 +3035,18 @@ bool TUdfNode::DoInit(TContext& ctx, ISource* src) {
 
     FunctionName = function->FuncName();
     ModuleName = function->ModuleName();
+    ScriptUdf = function->IsScript();
+    if (ScriptUdf && as_tuple->GetTupleSize() > 1) {
+        ctx.Error(Pos) << "Udf: user type is not supported for script udfs";
+        return false;
+    }
+
+    if (ScriptUdf) {
+        for (size_t i = 0; i < function->GetTupleSize(); ++i) {
+            ScriptArgs.push_back(function->GetTupleElement(i));
+        }
+    }
+
     TVector<TNodePtr> external;
     external.reserve(as_tuple->GetTupleSize() - 1);
 
@@ -3049,9 +3069,26 @@ bool TUdfNode::DoInit(TContext& ctx, ISource* src) {
     if (TStructNode* named_args = Args[1]->GetStructNode(); named_args) {
         for (const auto &arg: named_args->GetExprs()) {
             if (arg->GetLabel() == "TypeConfig") {
+                if (function->IsScript()) {
+                    ctx.Error() << "Udf: TypeConfig is not supported for script udfs";
+                    return false;
+                }
+
                 TypeConfig = MakeAtomFromExpression(Pos, ctx, arg);
             } else if (arg->GetLabel() == "RunConfig") {
+                if (function->IsScript()) {
+                    ctx.Error() << "Udf: RunConfig is not supported for script udfs";
+                    return false;
+                }
+
                 RunConfig = arg;
+            } else if (arg->GetLabel() == "Cpu") {
+                Cpu = MakeAtomFromExpression(Pos, ctx, arg);
+            } else if (arg->GetLabel() == "ExtraMem") {
+                ExtraMem = MakeAtomFromExpression(Pos, ctx, arg);
+            } else {
+                ctx.Error() << "Udf: unexpected named argument: " << arg->GetLabel();
+                return false;
             }
         }
     }
@@ -3079,6 +3116,31 @@ const TDeferredAtom& TUdfNode::GetTypeConfig() const {
     return TypeConfig;
 }
 
+TNodePtr TUdfNode::BuildOptions() const {
+    if (Cpu.Empty() && ExtraMem.Empty()) {
+        return nullptr;
+    }
+
+    auto options = Y();
+    if (!Cpu.Empty()) {
+        options = L(options, Q(Y(Q("cpu"), Cpu.Build())));
+    }
+
+    if (!ExtraMem.Empty()) {
+        options = L(options, Q(Y(Q("extraMem"), ExtraMem.Build())));
+    }
+
+    return Q(options);
+}
+
+bool TUdfNode::IsScript() const {
+    return ScriptUdf;
+}
+
+const TVector<TNodePtr>& TUdfNode::GetScriptArgs() const {
+    return ScriptArgs;
+}
+
 TUdfNode* TUdfNode::GetUdfNode() {
     return this;
 }
diff --git a/yql/essentials/sql/v1/node.h b/yql/essentials/sql/v1/node.h
index c197f7ac900..f3e3f5f5808 100644
--- a/yql/essentials/sql/v1/node.h
+++ b/yql/essentials/sql/v1/node.h
@@ -194,6 +194,7 @@ namespace NSQLTranslationV1 {
         virtual bool HasSelectResult() const;
         virtual const TString* FuncName() const;
         virtual const TString* ModuleName() const;
+        virtual bool IsScript() const;
         virtual bool HasSkip() const;
 
         virtual TColumnNode* GetColumnNode();
@@ -325,6 +326,7 @@ namespace NSQLTranslationV1 {
         virtual bool HasSelectResult() const override;
         virtual const TString* FuncName() const override;
         virtual const TString* ModuleName() const override;
+        virtual bool IsScript() const override;
         virtual bool HasSkip() const override;
 
         virtual TColumnNode* GetColumnNode() override;
@@ -950,6 +952,9 @@ namespace NSQLTranslationV1 {
         const TDeferredAtom& GetTypeConfig() const;
         TUdfNode* GetUdfNode() override;
         const TUdfNode* GetUdfNode() const override;
+        bool IsScript() const override;
+        const TVector<TNodePtr>& GetScriptArgs() const;
+        TNodePtr BuildOptions() const;
     private:
         TVector<TNodePtr> Args;
         const TString* FunctionName;
@@ -957,6 +962,10 @@ namespace NSQLTranslationV1 {
         TNodePtr ExternalTypesTuple = nullptr;
         TNodePtr RunConfig;
         TDeferredAtom TypeConfig;
+        TDeferredAtom Cpu;
+        TDeferredAtom ExtraMem;
+        bool ScriptUdf = false;
+        TVector<TNodePtr> ScriptArgs;
     };
 
     class IAggregation: public INode {
@@ -1511,6 +1520,12 @@ namespace NSQLTranslationV1 {
 
 
     // Implemented in builtin.cpp
+    TNodePtr BuildSqlCall(TContext& ctx, TPosition pos, const TString& module, const TString& name, const TVector<TNodePtr>& args,
+        TNodePtr positionalArgs, TNodePtr namedArgs, TNodePtr customUserType, const TDeferredAtom& typeConfig, TNodePtr runConfig,
+        TNodePtr options);
+    TNodePtr BuildScriptUdf(TPosition pos, const TString& moduleName, const TString& funcName, const TVector<TNodePtr>& args,
+        TNodePtr options);
+
     TNodePtr BuildCallable(TPosition pos, const TString& module, const TString& name, const TVector<TNodePtr>& args, bool forReduce = false);
     TNodePtr BuildUdf(TContext& ctx, TPosition pos, const TString& module, const TString& name, const TVector<TNodePtr>& args);
     TNodePtr BuildBuiltinFunc(
diff --git a/yql/essentials/sql/v1/sql_call_expr.cpp b/yql/essentials/sql/v1/sql_call_expr.cpp
index 1871c1bbc0b..f279e7db779 100644
--- a/yql/essentials/sql/v1/sql_call_expr.cpp
+++ b/yql/essentials/sql/v1/sql_call_expr.cpp
@@ -7,9 +7,6 @@
 
 namespace NSQLTranslationV1 {
 
-TNodePtr BuildSqlCall(TContext& ctx, TPosition pos, const TString& module, const TString& name, const TVector<TNodePtr>& args,
-    TNodePtr positionalArgs, TNodePtr namedArgs, TNodePtr customUserType, const TDeferredAtom& typeConfig, TNodePtr runConfig);
-
 using namespace NSQLv1Generated;
 
 static bool ValidateForCounters(const TString& input) {
@@ -49,26 +46,42 @@ TNodePtr TSqlCallExpr::BuildCall() {
             TNodePtr named = named_args->Y("TypeOf", named_args);
 
             TNodePtr custom_user_type = new TCallNodeImpl(Pos, "TupleType", {positional, named, udf_node->GetExternalTypes()});
+            TNodePtr options = udf_node->BuildOptions();
+
+            if (udf_node->IsScript()) {
+                auto udf = BuildScriptUdf(Pos, udf_node->GetModule(), udf_node->GetFunction(), udf_node->GetScriptArgs(), options);
+                TVector<TNodePtr> applyArgs;
+                applyArgs.push_back(new TAstAtomNodeImpl(Pos, !NamedArgs.empty() ? "NamedApply" : "Apply", TNodeFlags::Default));
+                applyArgs.push_back(udf);
+                if (!NamedArgs.empty()) {
+                    applyArgs.push_back(BuildTuple(Pos, PositionalArgs));
+                    applyArgs.push_back(BuildStructure(Pos, NamedArgs));
+                } else {
+                    applyArgs.insert(applyArgs.end(), PositionalArgs.begin(), PositionalArgs.end());
+                }
+
+                return new TAstListNodeImpl(Pos, applyArgs);
+            }
 
             return BuildSqlCall(Ctx, Pos, udf_node->GetModule(), udf_node->GetFunction(),
                                 args, positional_args, named_args, custom_user_type,
-                                udf_node->GetTypeConfig(), udf_node->GetRunConfig());
+                                udf_node->GetTypeConfig(), udf_node->GetRunConfig(), options);
         }
 
-        if (Node && !Node->FuncName()) {
+        if (Node && (!Node->FuncName() || Node->IsScript())) {
             Module = "YQL";
             Func = NamedArgs.empty() ? "Apply" : "NamedApply";
             warnOnYqlNameSpace = false;
             args.push_back(Node);
         }
 
-        if (Node && Node->FuncName()) {
+        if (Node && Node->FuncName() && !Node->IsScript()) {
             Module = Node->ModuleName() ? *Node->ModuleName() : "YQL";
             Func = *Node->FuncName();
         }
         bool mustUseNamed = !NamedArgs.empty();
         if (mustUseNamed) {
-            if (Node && !Node->FuncName()) {
+            if (Node && (!Node->FuncName() || Node->IsScript())) {
                 mustUseNamed = false;
             }
             args.emplace_back(BuildTuple(Pos, PositionalArgs));
@@ -276,7 +289,7 @@ bool TSqlCallExpr::FillArg(const TString& module, const TString& func, size_t& i
 bool TSqlCallExpr::FillArgs(const TRule_named_expr_list& node) {
     TString module = Module;
     TString func = Func;
-    if (Node && Node->FuncName()) {
+    if (Node && Node->FuncName() && !Node->IsScript()) {
         module = Node->ModuleName() ? *Node->ModuleName() : "YQL";
         func = *Node->FuncName();
     }
diff --git a/yql/essentials/sql/v1/sql_call_expr.h b/yql/essentials/sql/v1/sql_call_expr.h
index 9b9d39b81a8..f01ceabadfa 100644
--- a/yql/essentials/sql/v1/sql_call_expr.h
+++ b/yql/essentials/sql/v1/sql_call_expr.h
@@ -4,9 +4,6 @@
 
 namespace NSQLTranslationV1 {
 
-TNodePtr BuildSqlCall(TContext& ctx, TPosition pos, const TString& module, const TString& name, const TVector<TNodePtr>& args,
-    TNodePtr positionalArgs, TNodePtr namedArgs, TNodePtr customUserType, const TDeferredAtom& typeConfig, TNodePtr runConfig);
-
 using namespace NSQLv1Generated;
 
 class TSqlCallExpr: public TSqlTranslation {
diff --git a/yql/essentials/tests/sql/minirun/part0/canondata/result.json b/yql/essentials/tests/sql/minirun/part0/canondata/result.json
index 99503efb9b0..564c44698bf 100644
--- a/yql/essentials/tests/sql/minirun/part0/canondata/result.json
+++ b/yql/essentials/tests/sql/minirun/part0/canondata/result.json
@@ -1331,6 +1331,20 @@
             "uri": "https://{canondata_backend}/1946324/42941f53c7bc2748dcbf9469a617a6dd3e805076/resource.tar.gz#test.test_select-id_xor-default.txt-Results_/results.txt"
         }
     ],
+    "test.test[udf-udf_with_settings--Debug]": [
+        {
+            "checksum": "a7b364b692c35af7b468f42ab6a7b43f",
+            "size": 519,
+            "uri": "https://{canondata_backend}/1936273/248e207a8ecbe25b7fa36584b5664ede6d1e7705/resource.tar.gz#test.test_udf-udf_with_settings--Debug_/opt.yql"
+        }
+    ],
+    "test.test[udf-udf_with_settings--Results]": [
+        {
+            "checksum": "81bbe1cc55b32f1d4e823c7f81b0d3c5",
+            "size": 698,
+            "uri": "https://{canondata_backend}/1936273/248e207a8ecbe25b7fa36584b5664ede6d1e7705/resource.tar.gz#test.test_udf-udf_with_settings--Results_/results.txt"
+        }
+    ],
     "test.test[weak_field-weak_member_string_copy-default.txt-Debug]": [
         {
             "checksum": "23a485ccdd4e2b075a79544c2f95663f",
diff --git a/yql/essentials/tests/sql/minirun/part1/canondata/result.json b/yql/essentials/tests/sql/minirun/part1/canondata/result.json
index 95c90f0f006..74ea11e4a39 100644
--- a/yql/essentials/tests/sql/minirun/part1/canondata/result.json
+++ b/yql/essentials/tests/sql/minirun/part1/canondata/result.json
@@ -1272,6 +1272,20 @@
             "uri": "https://{canondata_backend}/995452/57f8b127ed5fa9fae2dd5ebb0f5870d86a7fcd2f/resource.tar.gz#test.test_udf-generic_udf--Results_/results.txt"
         }
     ],
+    "test.test[udf-python_udf_with_settings--Debug]": [
+        {
+            "checksum": "7642bb4ce4d7c5e888699fc134d84437",
+            "size": 600,
+            "uri": "https://{canondata_backend}/1936997/937db8f877d3f87af16df30b98be76320143ae68/resource.tar.gz#test.test_udf-python_udf_with_settings--Debug_/opt.yql"
+        }
+    ],
+    "test.test[udf-python_udf_with_settings--Results]": [
+        {
+            "checksum": "16c001ac88e22d3cb10094b198b6a209",
+            "size": 996,
+            "uri": "https://{canondata_backend}/1936997/937db8f877d3f87af16df30b98be76320143ae68/resource.tar.gz#test.test_udf-python_udf_with_settings--Results_/results.txt"
+        }
+    ],
     "test.test[udf-trivial_udf--Debug]": [
         {
             "checksum": "e30ef93274f818b56638089fa4a0513e",
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/result.json b/yql/essentials/tests/sql/sql2yql/canondata/result.json
index b5a310737c1..2328e0f0a6d 100644
--- a/yql/essentials/tests/sql/sql2yql/canondata/result.json
+++ b/yql/essentials/tests/sql/sql2yql/canondata/result.json
@@ -6957,6 +6957,13 @@
             "uri": "https://{canondata_backend}/1942173/99e88108149e222741552e7e6cddef041d6a2846/resource.tar.gz#test_sql2yql.test_udf-named_args_/sql.yql"
         }
     ],
+    "test_sql2yql.test[udf-python_udf_with_settings]": [
+        {
+            "checksum": "2a7556101ff528c09db2661b0593ed03",
+            "size": 1671,
+            "uri": "https://{canondata_backend}/1936273/4f1232af7b9d4a48fb0bf662681f0206a7380979/resource.tar.gz#test_sql2yql.test_udf-python_udf_with_settings_/sql.yql"
+        }
+    ],
     "test_sql2yql.test[udf-same_udf_modules]": [
         {
             "checksum": "218155c5b56f752fea69b11200019e25",
@@ -6999,6 +7006,13 @@
             "uri": "https://{canondata_backend}/1942173/99e88108149e222741552e7e6cddef041d6a2846/resource.tar.gz#test_sql2yql.test_udf-udf_result_member_/sql.yql"
         }
     ],
+    "test_sql2yql.test[udf-udf_with_settings]": [
+        {
+            "checksum": "7116564fc7e133352d9789d1ac74f435",
+            "size": 1221,
+            "uri": "https://{canondata_backend}/1936273/4f1232af7b9d4a48fb0bf662681f0206a7380979/resource.tar.gz#test_sql2yql.test_udf-udf_with_settings_/sql.yql"
+        }
+    ],
     "test_sql2yql.test[udf-wrong_args_fail]": [
         {
             "checksum": "7173a52302ba0a2e88761d1d44cbe14d",
@@ -10724,6 +10738,11 @@
             "uri": "file://test_sql_format.test_udf-named_args_/formatted.sql"
         }
     ],
+    "test_sql_format.test[udf-python_udf_with_settings]": [
+        {
+            "uri": "file://test_sql_format.test_udf-python_udf_with_settings_/formatted.sql"
+        }
+    ],
     "test_sql_format.test[udf-same_udf_modules]": [
         {
             "uri": "file://test_sql_format.test_udf-same_udf_modules_/formatted.sql"
@@ -10754,6 +10773,11 @@
             "uri": "file://test_sql_format.test_udf-udf_result_member_/formatted.sql"
         }
     ],
+    "test_sql_format.test[udf-udf_with_settings]": [
+        {
+            "uri": "file://test_sql_format.test_udf-udf_with_settings_/formatted.sql"
+        }
+    ],
     "test_sql_format.test[udf-wrong_args_fail]": [
         {
             "uri": "file://test_sql_format.test_udf-wrong_args_fail_/formatted.sql"
@@ -11247,6 +11271,27 @@
             "uri": "https://{canondata_backend}/1937429/434276f26b2857be3c5ad3fdbbf877d2bf775ac5/resource.tar.gz#test_sql_negative.test_udf-named_args_for_script_with_wrong_order-default.txt_/err_file.out"
         }
     ],
+    "test_sql_negative.test[udf-python_udf_with_settings_fail1-default.txt]": [
+        {
+            "checksum": "222536fdab78ce2d5977447e8790d7b6",
+            "size": 130,
+            "uri": "https://{canondata_backend}/1942278/6d832fede309393ea2fafff4a7beb49daf9842a1/resource.tar.gz#test_sql_negative.test_udf-python_udf_with_settings_fail1-default.txt_/err_file.out"
+        }
+    ],
+    "test_sql_negative.test[udf-python_udf_with_settings_fail2-default.txt]": [
+        {
+            "checksum": "9cf098b9b96d05ab8e0d837b21e5930b",
+            "size": 116,
+            "uri": "https://{canondata_backend}/1942278/6d832fede309393ea2fafff4a7beb49daf9842a1/resource.tar.gz#test_sql_negative.test_udf-python_udf_with_settings_fail2-default.txt_/err_file.out"
+        }
+    ],
+    "test_sql_negative.test[udf-python_udf_with_settings_fail3-default.txt]": [
+        {
+            "checksum": "97d780dab08cad5050ca810da93b7d93",
+            "size": 115,
+            "uri": "https://{canondata_backend}/1942278/6d832fede309393ea2fafff4a7beb49daf9842a1/resource.tar.gz#test_sql_negative.test_udf-python_udf_with_settings_fail3-default.txt_/err_file.out"
+        }
+    ],
     "test_sql_negative.test[union_all-union_all_with_top_level_limits-default.txt]": [
         {
             "checksum": "cef00afbcdfa2a07b6b3724865dc1a22",
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-python_udf_with_settings_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-python_udf_with_settings_/formatted.sql
new file mode 100644
index 00000000000..07293355942
--- /dev/null
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-python_udf_with_settings_/formatted.sql
@@ -0,0 +1,13 @@
+$f = Python::f(
+    Callable<(String, y: String) -> String>, @@
+def f(x,y):
+    return x + y
+@@
+);
+
+$f = Udf($f, '64.0' AS Cpu, '4294967296' AS ExtraMem);
+
+SELECT
+    $f('foo', '?'),
+    $f('foo', '!' AS y)
+;
diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-udf_with_settings_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-udf_with_settings_/formatted.sql
new file mode 100644
index 00000000000..87f7b263e13
--- /dev/null
+++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_udf-udf_with_settings_/formatted.sql
@@ -0,0 +1,6 @@
+$g = String::AsciiToUpper;
+$g = Udf($g, '64.0' AS Cpu, '4294967296' AS ExtraMem);
+
+SELECT
+    $g('foo')
+;
diff --git a/yql/essentials/tests/sql/suites/udf/python_udf_with_settings.cfg b/yql/essentials/tests/sql/suites/udf/python_udf_with_settings.cfg
new file mode 100644
index 00000000000..5fee00e9941
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/python_udf_with_settings.cfg
@@ -0,0 +1 @@
+udf python3_udf
diff --git a/yql/essentials/tests/sql/suites/udf/python_udf_with_settings.sql b/yql/essentials/tests/sql/suites/udf/python_udf_with_settings.sql
new file mode 100644
index 00000000000..10ec14a8648
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/python_udf_with_settings.sql
@@ -0,0 +1,10 @@
+$f = Python::f(Callable<(String,y:String)->String>,@@
+def f(x,y):
+    return x + y
+@@);
+
+$f = Udf($f, "64.0" as Cpu, "4294967296" as ExtraMem);
+
+SELECT
+    $f('foo', '?'),
+    $f('foo', '!' as y)
diff --git a/yql/essentials/tests/sql/suites/udf/python_udf_with_settings_fail1.sqlx b/yql/essentials/tests/sql/suites/udf/python_udf_with_settings_fail1.sqlx
new file mode 100644
index 00000000000..b753a203f0b
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/python_udf_with_settings_fail1.sqlx
@@ -0,0 +1,9 @@
+$f = Python::f(Callable<(String,y:String)->String>,@@
+def f(x,y):
+    return x + y
+@@);
+
+$f = Udf($f, Int32); -- no custom user type
+
+SELECT
+    $f('foo', '?')
diff --git a/yql/essentials/tests/sql/suites/udf/python_udf_with_settings_fail2.sqlx b/yql/essentials/tests/sql/suites/udf/python_udf_with_settings_fail2.sqlx
new file mode 100644
index 00000000000..df08959e4e9
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/python_udf_with_settings_fail2.sqlx
@@ -0,0 +1,9 @@
+$f = Python::f(Callable<(String,y:String)->String>,@@
+def f(x,y):
+    return x + y
+@@);
+
+$f = Udf($f, "bar" as TypeConfig); -- no type config
+
+SELECT
+    $f('foo', '?')
diff --git a/yql/essentials/tests/sql/suites/udf/python_udf_with_settings_fail3.sqlx b/yql/essentials/tests/sql/suites/udf/python_udf_with_settings_fail3.sqlx
new file mode 100644
index 00000000000..d27b2ba6a36
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/python_udf_with_settings_fail3.sqlx
@@ -0,0 +1,9 @@
+$f = Python::f(Callable<(String,y:String)->String>,@@
+def f(x,y):
+    return x + y
+@@);
+
+$f = Udf($f, "bar" as RunConfig); -- no run config
+
+SELECT
+    $f('foo', '?')
diff --git a/yql/essentials/tests/sql/suites/udf/udf_with_settings.cfg b/yql/essentials/tests/sql/suites/udf/udf_with_settings.cfg
new file mode 100644
index 00000000000..f75b50116da
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/udf_with_settings.cfg
@@ -0,0 +1,2 @@
+udf string_udf
+
diff --git a/yql/essentials/tests/sql/suites/udf/udf_with_settings.sql b/yql/essentials/tests/sql/suites/udf/udf_with_settings.sql
new file mode 100644
index 00000000000..adb7067455a
--- /dev/null
+++ b/yql/essentials/tests/sql/suites/udf/udf_with_settings.sql
@@ -0,0 +1,6 @@
+$g = String::AsciiToUpper;
+
+$g = Udf($g, "64.0" as Cpu, "4294967296" as ExtraMem);
+
+SELECT
+    $g('foo')