better support of placeholders inside library url - in SQL parser and parameters extraction

9 files changed, 149 insertions, 70 deletions

diff --git a/CMakeLists.darwin.txt b/CMakeLists.darwin.txt
index 965d9b971b..525e871e20 100644
--- a/CMakeLists.darwin.txt
+++ b/CMakeLists.darwin.txt
@@ -494,11 +494,11 @@ add_subdirectory(ydb/library/yql/providers/common/mkql)
 add_subdirectory(ydb/library/yql/ast)
 add_subdirectory(library/cpp/containers/sorted_vector)
 add_subdirectory(library/cpp/string_utils/levenshtein_diff)
+add_subdirectory(library/cpp/yson/node)
+add_subdirectory(library/cpp/yson/json)
 add_subdirectory(ydb/library/yql/core/issue)
 add_subdirectory(ydb/library/yql/public/issue)
 add_subdirectory(ydb/library/yql/core)
-add_subdirectory(library/cpp/yson/node)
-add_subdirectory(library/cpp/yson/json)
 add_subdirectory(ydb/library/yql/core/file_storage)
 add_subdirectory(library/cpp/logger/global)
 add_subdirectory(library/cpp/regex/pcre)
diff --git a/CMakeLists.linux.txt b/CMakeLists.linux.txt
index 858a8ce614..91fe27b3bb 100644
--- a/CMakeLists.linux.txt
+++ b/CMakeLists.linux.txt
@@ -498,11 +498,11 @@ add_subdirectory(ydb/library/yql/providers/common/mkql)
 add_subdirectory(ydb/library/yql/ast)
 add_subdirectory(library/cpp/containers/sorted_vector)
 add_subdirectory(library/cpp/string_utils/levenshtein_diff)
+add_subdirectory(library/cpp/yson/node)
+add_subdirectory(library/cpp/yson/json)
 add_subdirectory(ydb/library/yql/core/issue)
 add_subdirectory(ydb/library/yql/public/issue)
 add_subdirectory(ydb/library/yql/core)
-add_subdirectory(library/cpp/yson/node)
-add_subdirectory(library/cpp/yson/json)
 add_subdirectory(ydb/library/yql/core/file_storage)
 add_subdirectory(library/cpp/logger/global)
 add_subdirectory(library/cpp/regex/pcre)
diff --git a/ydb/library/yql/ast/CMakeLists.txt b/ydb/library/yql/ast/CMakeLists.txt
index 3968bd3ef0..835775a300 100644
--- a/ydb/library/yql/ast/CMakeLists.txt
+++ b/ydb/library/yql/ast/CMakeLists.txt
@@ -18,6 +18,7 @@ target_link_libraries(library-yql-ast PUBLIC
   library-cpp-enumbitset
   cpp-string_utils-levenshtein_diff
   library-cpp-yson
+  cpp-yson-node
   yql-public-udf
   library-yql-utils
   yql-core-issue
diff --git a/ydb/library/yql/ast/yql_expr.cpp b/ydb/library/yql/ast/yql_expr.cpp
index c40d1a9812..6c72acf1c3 100644
--- a/ydb/library/yql/ast/yql_expr.cpp
+++ b/ydb/library/yql/ast/yql_expr.cpp
@@ -3,6 +3,7 @@
 #include "yql_gc_nodes.h"
 
 #include <ydb/library/yql/utils/utf8.h>
+#include <ydb/library/yql/core/issue/yql_issue.h>
 
 #include <ydb/library/yql/parser/pg_catalog/catalog.h>
 #include <library/cpp/containers/stack_vector/stack_vec.h>
@@ -116,6 +117,12 @@ namespace {
             Expr.AddError(TIssue(node.GetPosition(), message));
         }
 
+        void AddInfo(const TAstNode& node, const TString& message) {
+            auto issue = TIssue(node.GetPosition(), message);
+            issue.SetCode(TIssuesIds::INFO, TSeverityIds::S_INFO);
+            Expr.AddError(issue);
+        }
+
         TExprNode::TPtr&& ProcessNode(const TAstNode& node, TExprNode::TPtr&& exprNode) {
             if (TypeAnnotationIndex != Max<ui32>()) {
                 exprNode->SetTypeAnn(CompileTypeAnnotation(node));
@@ -1057,7 +1064,7 @@ namespace {
         return true;
     }
 
-    bool CompileDeclare(const TAstNode& node, TContext& ctx) {
+    bool CompileDeclare(const TAstNode& node, TContext& ctx, bool checkOnly) {
         if (node.GetChildrenCount() != 3) {
             ctx.AddError(node, "Expected list of size 3");
             return false;
@@ -1090,9 +1097,25 @@ namespace {
                 std::move(typeExpr.front())
             }));
 
-        if (!ctx.Frames.back().Bindings.emplace(nameStr, TExprNode::TListType{std::move(parameterExpr)}).second) {
-            ctx.AddError(node, TStringBuilder() << "Declare statement hides previously defined name: "
-                << nameStr);
+        bool error = false;
+        if (checkOnly) {
+            auto it = ctx.Frames.back().Bindings.find(nameStr);
+            if (it == ctx.Frames.back().Bindings.end()) {
+                ctx.AddError(*name, TStringBuilder() << "Missing parameter: " << nameStr);
+                return false;
+            }
+
+            if (it->second.size() != 1 || !it->second.front()->IsCallable("Parameter")) {
+                error = true;
+            }
+        } else {
+            if (!ctx.Frames.back().Bindings.emplace(nameStr, TExprNode::TListType{ std::move(parameterExpr) }).second) {
+                error = true;
+            }
+        }
+
+        if (error) {
+            ctx.AddError(node, TStringBuilder() << "Declare statement hides previously defined name: " << nameStr);
             return false;
         }
 
@@ -1122,9 +1145,32 @@ namespace {
             url = file->GetContent();
         }
 
+        if (url) {
+            auto world = ctx.FindBinding("world");
+            if (!world.empty()) {
+                TSet<TString> names;
+                SubstParameters(url, Nothing(), &names);
+                for (const auto& name : names) {
+                    auto nameRef = ctx.FindBinding(name);
+                    if (nameRef.empty()) {
+                        ctx.AddError(node, TStringBuilder() << "Name not found: " << name);
+                        return false;
+                    }
+
+                    TExprNode::TListType args = world;
+                    args.insert(args.end(), nameRef.begin(), nameRef.end());
+                    auto newWorld = TExprNode::TListType{ ctx.Expr.NewCallable(node.GetPosition(), "Left!", {
+                        ctx.Expr.NewCallable(node.GetPosition(), "Cons!", std::move(args)) })};
+
+                    ctx.Frames.back().Bindings["world"] = newWorld;
+                    world = newWorld;
+                }
+            }
+        }
+
         if (!ctx.ModuleResolver) {
-            ctx.AddError(*name, "Module resolver isn't available");
-            return false;
+            ctx.AddInfo(*name, "Module resolver isn't available");
+            return true;
         }
 
         if (url) {
@@ -1196,6 +1242,9 @@ namespace {
                 } else if (firstChild->GetContent() == TStringBuf("set_package_version")) {
                     if (!CompileSetPackageVersion(*node, ctx))
                         return {};
+                } else if (firstChild->GetContent() == TStringBuf("declare")) {
+                    if (!CompileDeclare(*node, ctx, false))
+                        return {};
                 }
             }
 
@@ -1247,8 +1296,10 @@ namespace {
                     return {};
                 }
 
-                if (!CompileDeclare(*node, ctx))
+                if (!CompileDeclare(*node, ctx, true))
                     return {};
+
+                continue;
             } else if (firstChild->GetContent() == TStringBuf("library")) {
                 if (!topLevel) {
                     ctx.AddError(*firstChild, "Library statements are only allowed on top level block");
@@ -3254,6 +3305,73 @@ void CheckCounts(const TExprNode& root) {
     CheckReferences(root, refCounts, visited);
 }
 
+TString SubstParameters(const TString& str, const TMaybe<NYT::TNode>& params, TSet<TString>* usedNames) {
+    size_t pos = 0;
+    try {
+        TStringBuilder res;
+        bool insideBrackets = false;
+        TStringBuilder paramBuilder;
+        for (char c : str) {
+            if (c == '{') {
+                if (insideBrackets) {
+                    throw yexception() << "Unpexpected {";
+                }
+
+                insideBrackets = true;
+                continue;
+            }
+
+            if (c == '}') {
+                if (!insideBrackets) {
+                    throw yexception() << "Unexpected }";
+                }
+
+                insideBrackets = false;
+                TString param = paramBuilder;
+                paramBuilder.clear();
+                if (usedNames) {
+                    usedNames->insert(param);
+                }
+
+                if (params) {
+                    const auto& map = params->AsMap();
+                    auto it = map.find(param);
+                    if (it == map.end()) {
+                        throw yexception() << "No such parameter: '" << param << "'";
+                    }
+
+                    const auto& value = it->second["Data"];
+                    if (!value.IsString()) {
+                        throw yexception() << "Parameter value must be a string";
+                    }
+
+                    res << value.AsString();
+                }
+
+                continue;
+            }
+
+            if (insideBrackets) {
+                paramBuilder << c;
+            }
+            else {
+                res << c;
+            }
+
+            ++pos;
+        }
+
+        if (insideBrackets) {
+            throw yexception() << "Missing }";
+        }
+
+        return res;
+    }
+    catch (yexception& e) {
+        throw yexception() << "Failed to substitute parameters into url: " << str << ", reason:" << e.what() << ", position: " << pos;
+    }
+}
+
 } // namespace NYql
 
 template<>
diff --git a/ydb/library/yql/ast/yql_expr.h b/ydb/library/yql/ast/yql_expr.h
index ab66d5da6c..ae463ad730 100644
--- a/ydb/library/yql/ast/yql_expr.h
+++ b/ydb/library/yql/ast/yql_expr.h
@@ -12,6 +12,8 @@
 #include <ydb/library/yql/public/issue/yql_issue_manager.h>
 #include <ydb/library/yql/public/udf/udf_data_type.h>
 
+#include <library/cpp/yson/node/node.h>
+
 #include <library/cpp/string_utils/levenshtein_diff/levenshtein_diff.h>
 #include <library/cpp/enumbitset/enumbitset.h>
 #include <library/cpp/containers/stack_vector/stack_vec.h>
@@ -23,6 +25,7 @@
 #include <util/generic/cast.h>
 #include <util/generic/hash.h>
 #include <util/generic/maybe.h>
+#include <util/generic/set.h>
 #include <util/generic/bt_exception.h>
 #include <util/generic/algorithm.h>
 #include <util/digest/murmur.h>
@@ -2622,6 +2625,8 @@ TAstParseResult ConvertToAst(const TExprNode& root, TExprContext& ctx, ui32 anno
 
 TExprNode::TListType GetLambdaBody(const TExprNode& lambda);
 
+TString SubstParameters(const TString& str, const TMaybe<NYT::TNode>& params, TSet<TString>* usedNames);
+
 } // namespace NYql
 
 template<>
diff --git a/ydb/library/yql/core/facade/yql_facade.cpp b/ydb/library/yql/core/facade/yql_facade.cpp
index 5b9fb5bd08..a758f81264 100644
--- a/ydb/library/yql/core/facade/yql_facade.cpp
+++ b/ydb/library/yql/core/facade/yql_facade.cpp
@@ -420,7 +420,7 @@ bool TProgram::ParseSql(const NSQLTranslation::TTranslationSettings& settings)
     return FillParseResult(SqlToYql(SourceCode_, settings, &warningRules), &warningRules);
 }
 
-bool TProgram::Compile(const TString& username) {
+bool TProgram::Compile(const TString& username, bool skipLibraries) {
     YQL_PROFILE_FUNC(TRACE);
 
     Y_ENSURE(AstRoot_, "Program not parsed yet");
@@ -432,7 +432,7 @@ bool TProgram::Compile(const TString& username) {
         return false;
     }
     TypeCtx_->IsReadOnly = true;
-    if (Modules_.get()) {
+    if (!skipLibraries && Modules_.get()) {
         auto libs = UserDataStorage_->GetLibraries();
         for (auto lib : libs) {
             if (!Modules_->AddFromFile(lib, *ExprCtx_, SyntaxVersion_, 0)) {
@@ -441,7 +441,7 @@ bool TProgram::Compile(const TString& username) {
         }
     }
 
-    if (!CompileExpr(*AstRoot_, ExprRoot_, *ExprCtx_, Modules_.get(), 0, SyntaxVersion_)) {
+    if (!CompileExpr(*AstRoot_, ExprRoot_, *ExprCtx_, skipLibraries ? nullptr : Modules_.get(), 0, SyntaxVersion_)) {
         return false;
     }
 
diff --git a/ydb/library/yql/core/facade/yql_facade.h b/ydb/library/yql/core/facade/yql_facade.h
index 82f8f1c22d..332d17b9cf 100644
--- a/ydb/library/yql/core/facade/yql_facade.h
+++ b/ydb/library/yql/core/facade/yql_facade.h
@@ -101,7 +101,7 @@ public:
     bool ParseSql();
     bool ParseSql(const NSQLTranslation::TTranslationSettings& settings);
 
-    bool Compile(const TString& username);
+    bool Compile(const TString& username, bool skipLibraries = false);
 
     TStatus Discover(const TString& username);
 
diff --git a/ydb/library/yql/core/yql_type_annotation.cpp b/ydb/library/yql/core/yql_type_annotation.cpp
index 0785e04c19..743210dd64 100644
--- a/ydb/library/yql/core/yql_type_annotation.cpp
+++ b/ydb/library/yql/core/yql_type_annotation.cpp
@@ -452,61 +452,7 @@ TString TModuleResolver::SubstParameters(const TString& str) {
         return str;
     }
 
-    size_t pos = 0;
-    try {
-        TStringBuilder res;
-        bool insideBrackets = false;
-        TStringBuilder paramBuilder;
-        for (char c : str) {
-            if (c == '{') {
-                if (insideBrackets) {
-                    throw yexception() << "Unpexpected {";
-                }
-
-                insideBrackets = true;
-                continue;
-            }
-
-            if (c == '}') {
-                if (!insideBrackets) {
-                    throw yexception() << "Unexpected }";
-                }
-
-                insideBrackets = false;
-                TString param = paramBuilder;
-                paramBuilder.clear();
-                const auto& map = Parameters->AsMap();
-                auto it = map.find(param);
-                if (it == map.end()) {
-                    throw yexception() << "No such parameter: '" << param << "'";
-                }
-
-                const auto& value = it->second["Data"];
-                if (!value.IsString()) {
-                    throw yexception() << "Parameter value must be a string";
-                }
-
-                res << value.AsString();
-                continue;
-            }
-
-            if (insideBrackets) {
-                paramBuilder << c;
-            } else {
-                res << c;
-            }
-
-            ++pos;
-        }
-
-        if (insideBrackets) {
-            throw yexception() << "Missing }";
-        }
-
-        return res;
-    } catch (yexception& e) {
-        throw yexception() << "Failed to substitute parameters into url: " << str << ", reason:" << e.what() << ", position: " << pos;
-    }
+    return ::NYql::SubstParameters(str, Parameters, nullptr);
 }
 
 } // namespace NYql
diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp
index 509586f573..9f0e83c570 100644
--- a/ydb/library/yql/sql/v1/sql.cpp
+++ b/ydb/library/yql/sql/v1/sql.cpp
@@ -9574,6 +9574,15 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
                     Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
                     return{};
                 }
+
+                TSet<TString> names;
+                SubstParameters(*file, Nothing(), &names);
+                for (const auto& name : names) {
+                    auto namedNode = GetNamedNode(name);
+                    if (!namedNode) {
+                        return{};
+                    }
+                }
             }
 
             Ctx.Libraries[alias]=file;