YQL-15828 Keep and use position and library name on errors in library.

8 files changed, 66 insertions, 55 deletions

diff --git a/ydb/library/yql/ast/yql_expr.cpp b/ydb/library/yql/ast/yql_expr.cpp
index 7f72763fc7a..12108ab7692 100644
--- a/ydb/library/yql/ast/yql_expr.cpp
+++ b/ydb/library/yql/ast/yql_expr.cpp
@@ -1238,11 +1238,11 @@ namespace {
         }
 
         if (url) {
-            if (!ctx.ModuleResolver->AddFromUrl(name->GetContent(), url, token, ctx.Expr, ctx.SyntaxVersion, 0)) {
+            if (!ctx.ModuleResolver->AddFromUrl(name->GetContent(), url, token, ctx.Expr, ctx.SyntaxVersion, 0, name->GetPosition())) {
                 return false;
             }
         } else {
-            if (!ctx.ModuleResolver->AddFromFile(name->GetContent(), ctx.Expr, ctx.SyntaxVersion, 0)) {
+            if (!ctx.ModuleResolver->AddFromFile(name->GetContent(), ctx.Expr, ctx.SyntaxVersion, 0, name->GetPosition())) {
                 return false;
             }
         }
diff --git a/ydb/library/yql/ast/yql_expr.h b/ydb/library/yql/ast/yql_expr.h
index 03e723045eb..943c38305be 100644
--- a/ydb/library/yql/ast/yql_expr.h
+++ b/ydb/library/yql/ast/yql_expr.h
@@ -2173,10 +2173,10 @@ using TModulesTable = THashMap<TString, TExportTable>;
 class IModuleResolver {
 public:
     typedef std::shared_ptr<IModuleResolver> TPtr;
-    virtual bool AddFromFile(const TStringBuf& file, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) = 0;
-    virtual bool AddFromUrl(const TStringBuf& file, const TStringBuf& url, const TStringBuf& tokenName, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) = 0;
-    virtual bool AddFromMemory(const TStringBuf& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) = 0;
-    virtual bool AddFromMemory(const TStringBuf& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TString& moduleName, std::vector<TString>* exports = nullptr, std::vector<TString>* imports = nullptr) = 0;
+    virtual bool AddFromFile(const std::string_view& file, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TPosition pos = {}) = 0;
+    virtual bool AddFromUrl(const std::string_view& file, const std::string_view& url, const std::string_view& tokenName, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TPosition pos = {}) = 0;
+    virtual bool AddFromMemory(const std::string_view& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TPosition pos = {}) = 0;
+    virtual bool AddFromMemory(const std::string_view& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TPosition pos, TString& moduleName, std::vector<TString>* exports = nullptr, std::vector<TString>* imports = nullptr) = 0;
     virtual bool Link(TExprContext& ctx) = 0;
     virtual void UpdateNextUniqueId(TExprContext& ctx) const = 0;
     virtual ui64 GetNextUniqueId() const = 0;
diff --git a/ydb/library/yql/core/yql_type_annotation.cpp b/ydb/library/yql/core/yql_type_annotation.cpp
index f9f4cbc6601..5a76e4f3a79 100644
--- a/ydb/library/yql/core/yql_type_annotation.cpp
+++ b/ydb/library/yql/core/yql_type_annotation.cpp
@@ -184,9 +184,9 @@ const TExportTable* TModuleResolver::GetModule(const TString& module) const {
     return Modules.FindPtr(normalizedModuleName);
 }
 
-bool TModuleResolver::AddFromUrl(const TStringBuf& file, const TStringBuf& url, const TStringBuf& tokenName, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) {
+bool TModuleResolver::AddFromUrl(const std::string_view& file, const std::string_view& url, const std::string_view& tokenName, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TPosition pos) {
     if (!UserData) {
-        ctx.AddError(TIssue(TPosition(), "Loading libraries is prohibited"));
+        ctx.AddError(TIssue(pos, "Loading libraries is prohibited"));
         return false;
     }
 
@@ -194,41 +194,41 @@ bool TModuleResolver::AddFromUrl(const TStringBuf& file, const TStringBuf& url,
     block.Type = EUserDataType::URL;
     block.Data = url;
     block.Data = SubstParameters(block.Data);
-    if (tokenName) {
+    if (!tokenName.empty()) {
         if (!Credentials) {
-            ctx.AddError(TIssue(TPosition(), "Missing credentials"));
+            ctx.AddError(TIssue(pos, "Missing credentials"));
             return false;
         }
         auto cred = Credentials->FindCredential(tokenName);
         if (!cred) {
-            ctx.AddError(TIssue(TPosition(), TStringBuilder() << "Unknown token name: " << tokenName));
+            ctx.AddError(TIssue(pos, TStringBuilder() << "Unknown token name: " << tokenName));
             return false;
         }
         block.UrlToken = cred->Content;
     }
     UserData->AddUserDataBlock(file, block);
 
-    return AddFromFile(file, ctx, syntaxVersion, packageVersion);
+    return AddFromFile(file, ctx, syntaxVersion, packageVersion, pos);
 }
 
-bool TModuleResolver::AddFromFile(const TStringBuf& file, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) {
+bool TModuleResolver::AddFromFile(const std::string_view& file, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TPosition pos) {
     if (!UserData) {
-        ctx.AddError(TIssue(TPosition(), "Loading libraries is prohibited"));
+        ctx.AddError(TIssue(pos, "Loading libraries is prohibited"));
         return false;
     }
 
     const auto fullName = TUserDataStorage::MakeFullName(file);
-    bool isSql = file.EndsWith(".sql");
-    bool isYql = file.EndsWith(".yql");
+    const bool isSql = file.ends_with(".sql");
+    const bool isYql = file.ends_with(".yql");
     if (!isSql && !isYql) {
-        ctx.AddError(TIssue(TStringBuilder() << "Unsupported syntax of library file, expected one of (.sql, .yql): " << file));
+        ctx.AddError(TIssue(pos, TStringBuilder() << "Unsupported syntax of library file, expected one of (.sql, .yql): " << file));
         return false;
     }
 
     const TUserDataBlock* block = UserData->FindUserDataBlock(fullName);
 
     if (!block) {
-        ctx.AddError(TIssue(TStringBuilder() << "File not found: " << file));
+        ctx.AddError(TIssue(pos, TStringBuilder() << "File not found: " << file));
         return false;
     }
 
@@ -252,7 +252,7 @@ bool TModuleResolver::AddFromFile(const TStringBuf& file, TExprContext& ctx, ui1
         break;
     case EUserDataType::URL:
         if (!UrlLoader) {
-            ctx.AddError(TIssue(TStringBuilder() << "Unable to load file \"" << file
+            ctx.AddError(TIssue(pos, TStringBuilder() << "Unable to load file \"" << file
                 << "\" from url, because url loader is not available"));
             return false;
         }
@@ -263,20 +263,20 @@ bool TModuleResolver::AddFromFile(const TStringBuf& file, TExprContext& ctx, ui1
         throw yexception() << "Unknown block type " << block->Type;
     }
 
-    return AddFromMemory(fullName, moduleName, isYql, body, ctx, syntaxVersion, packageVersion);
+    return AddFromMemory(fullName, moduleName, isYql, body, ctx, syntaxVersion, packageVersion, pos);
 }
 
-bool TModuleResolver::AddFromMemory(const TStringBuf& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) {
+bool TModuleResolver::AddFromMemory(const std::string_view& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TPosition pos) {
     TString unusedModuleName;
-    return AddFromMemory(file, body, ctx, syntaxVersion, packageVersion, unusedModuleName);
+    return AddFromMemory(file, body, ctx, syntaxVersion, packageVersion, pos, unusedModuleName);
 }
 
-bool TModuleResolver::AddFromMemory(const TStringBuf& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TString& moduleName, std::vector<TString>* exports, std::vector<TString>* imports) {
+bool TModuleResolver::AddFromMemory(const std::string_view& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TPosition pos, TString& moduleName, std::vector<TString>* exports, std::vector<TString>* imports) {
     const auto fullName = TUserDataStorage::MakeFullName(file);
-    bool isSql = file.EndsWith(".sql");
-    bool isYql = file.EndsWith(".yql");
+    const bool isSql = file.ends_with(".sql");
+    const bool isYql = file.ends_with(".yql");
     if (!isSql && !isYql) {
-        ctx.AddError(TIssue(TStringBuilder() << "Unsupported syntax of library file, expected one of (.sql, .yql): " << file));
+        ctx.AddError(TIssue(pos, TStringBuilder() << "Unsupported syntax of library file, expected one of (.sql, .yql): " << file));
         return false;
     }
 
@@ -290,16 +290,22 @@ bool TModuleResolver::AddFromMemory(const TStringBuf& file, const TString& body,
         }
     }
 
-    return AddFromMemory(fullName, moduleName, isYql, body, ctx, syntaxVersion, packageVersion, exports, imports);
+    return AddFromMemory(fullName, moduleName, isYql, body, ctx, syntaxVersion, packageVersion, pos, exports, imports);
 }
 
-bool TModuleResolver::AddFromMemory(const TString& fullName, const TString& moduleName, bool isYql, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, std::vector<TString>* exports, std::vector<TString>* imports) {
+bool TModuleResolver::AddFromMemory(const TString& fullName, const TString& moduleName, bool isYql, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TPosition pos, std::vector<TString>* exports, std::vector<TString>* imports) {
+    const auto addSubIssues = [&fullName](TIssue&& issue, const TIssues& issues) {
+        std::for_each(issues.begin(), issues.end(), [&](const TIssue& i) {
+            issue.AddSubIssue(MakeIntrusive<TIssue>(TPosition(i.Position.Column, i.Position.Row, fullName), i.GetMessage()));
+        });
+        return std::move(issue);
+    };
+
     TAstParseResult astRes;
     if (isYql) {
         astRes = ParseAst(body, nullptr, fullName);
         if (!astRes.IsOk()) {
-            ctx.IssueManager.AddIssues(astRes.Issues);
-            ctx.AddError(TIssue(TStringBuilder() << "Failed to parse YQL: " << fullName));
+            ctx.AddError(addSubIssues(TIssue(pos, TStringBuilder() << "Failed to parse YQL: " << fullName), astRes.Issues));
             return false;
         }
     } else {
@@ -312,23 +318,20 @@ bool TModuleResolver::AddFromMemory(const TString& fullName, const TString& modu
         settings.V0Behavior = NSQLTranslation::EV0Behavior::Silent;
         astRes = SqlToYql(body, settings);
         if (!astRes.IsOk()) {
-            ctx.IssueManager.AddIssues(astRes.Issues);
-            ctx.AddError(TIssue(TStringBuilder() << "Failed to parse SQL: " << fullName));
+            ctx.AddError(addSubIssues(TIssue(pos, TStringBuilder() << "Failed to parse SQL: " << fullName), astRes.Issues));
             return false;
         }
     }
 
     TLibraryCohesion cohesion;
     if (!CompileExpr(*astRes.Root, cohesion, LibsContext)) {
-        ctx.IssueManager.AddIssues(LibsContext.IssueManager.GetIssues());
-        ctx.AddError(TIssue(TStringBuilder() << "Failed to compile: " << fullName));
+        ctx.AddError(addSubIssues(TIssue(pos, TStringBuilder() << "Failed to compile: " << fullName), LibsContext.IssueManager.GetIssues()));
         return false;
     }
 
     if (OptimizeLibraries) {
         if (!OptimizeLibrary(cohesion, LibsContext)) {
-            ctx.IssueManager.AddIssues(LibsContext.IssueManager.GetIssues());
-            ctx.AddError(TIssue(TStringBuilder() << "Failed to optimize: " << fullName));
+            ctx.AddError(addSubIssues(TIssue(pos, TStringBuilder() << "Failed to optimize: " << fullName), LibsContext.IssueManager.GetIssues()));
             return false;
         }
     }
diff --git a/ydb/library/yql/core/yql_type_annotation.h b/ydb/library/yql/core/yql_type_annotation.h
index 7e63c2780df..863f8e2bd82 100644
--- a/ydb/library/yql/core/yql_type_annotation.h
+++ b/ydb/library/yql/core/yql_type_annotation.h
@@ -83,17 +83,17 @@ public:
     void RegisterPackage(const TString& package) override;
     bool SetPackageDefaultVersion(const TString& package, ui32 version) override;
     const TExportTable* GetModule(const TString& module) const override;
-    bool AddFromFile(const TStringBuf& file, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) override;
-    bool AddFromUrl(const TStringBuf& file, const TStringBuf& url, const TStringBuf& tokenName, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) override;
-    bool AddFromMemory(const TStringBuf& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion) override;
-    bool AddFromMemory(const TStringBuf& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TString& moduleName, std::vector<TString>* exports = nullptr, std::vector<TString>* imports = nullptr) override;
+    bool AddFromFile(const std::string_view& file, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TPosition pos) final;
+    bool AddFromUrl(const std::string_view& file, const std::string_view& url, const std::string_view& tokenName, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TPosition pos) final;
+    bool AddFromMemory(const std::string_view& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TPosition pos) final;
+    bool AddFromMemory(const std::string_view& file, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TPosition pos, TString& moduleName, std::vector<TString>* exports = nullptr, std::vector<TString>* imports = nullptr) final;
     bool Link(TExprContext& ctx) override;
     void UpdateNextUniqueId(TExprContext& ctx) const override;
     ui64 GetNextUniqueId() const override;
     IModuleResolver::TPtr CreateMutableChild() const override;
 
 private:
-    bool AddFromMemory(const TString& fullName, const TString& moduleName, bool isYql, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, std::vector<TString>* exports = nullptr, std::vector<TString>* imports = nullptr);
+    bool AddFromMemory(const TString& fullName, const TString& moduleName, bool isYql, const TString& body, TExprContext& ctx, ui16 syntaxVersion, ui32 packageVersion, TPosition pos, std::vector<TString>* exports = nullptr, std::vector<TString>* imports = nullptr);
     THashMap<TString, TLibraryCohesion> FilterLibsByVersion() const;
     static TString ExtractPackageNameFromModule(TStringBuf moduleName);
     TString SubstParameters(const TString& str);
diff --git a/ydb/library/yql/sql/v1/context.cpp b/ydb/library/yql/sql/v1/context.cpp
index a82b3de6d9f..c7f95e4863c 100644
--- a/ydb/library/yql/sql/v1/context.cpp
+++ b/ydb/library/yql/sql/v1/context.cpp
@@ -86,7 +86,7 @@ TContext::TContext(const NSQLTranslation::TTranslationSettings& settings,
     , AnsiQuotedIdentifiers(settings.AnsiLexer)
 {
     for (auto lib : settings.Libraries) {
-        Libraries[lib] = Nothing();
+        Libraries.emplace(lib, TLibraryStuff());
     }
 
     Scoped = MakeIntrusive<TScopedState>();
diff --git a/ydb/library/yql/sql/v1/context.h b/ydb/library/yql/sql/v1/context.h
index 60e4d74cee9..3ff7ccbe478 100644
--- a/ydb/library/yql/sql/v1/context.h
+++ b/ydb/library/yql/sql/v1/context.h
@@ -273,7 +273,9 @@ namespace NSQLTranslationV1 {
         // see YQL-10265
         bool AnsiCurrentRow = false;
         TMaybe<bool> YsonCastToString;
-        THashMap<TString, TMaybe<std::pair<TString, TString>>> Libraries; // alias -> optional file with token
+        using TLiteralWithPosition = std::pair<TString, TPosition>;
+        using TLibraryStuff = std::tuple<TPosition, std::optional<TLiteralWithPosition>, std::optional<TLiteralWithPosition>>;
+        std::unordered_map<TString, TLibraryStuff> Libraries; // alias -> optional file with token
         THashMap<TString, ui32> PackageVersions;
         NYql::TWarningPolicy WarningPolicy;
         TString PqReadByRtmrCluster;
diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp
index 1237efa505d..b63fa214a69 100644
--- a/ydb/library/yql/sql/v1/query.cpp
+++ b/ydb/library/yql/sql/v1/query.cpp
@@ -2215,11 +2215,11 @@ public:
             }
 
             for (const auto& lib : ctx.Libraries) {
-                auto node = Y("library", new TAstAtomNodeImpl(Pos, lib.first, TNodeFlags::ArbitraryContent));
-                if (lib.second) {
-                    node = L(node, new TAstAtomNodeImpl(Pos, lib.second->first, TNodeFlags::ArbitraryContent));
-                    if (lib.second->second) {
-                        node = L(node, new TAstAtomNodeImpl(Pos, lib.second->second, TNodeFlags::ArbitraryContent));
+                auto node = Y("library", new TAstAtomNodeImpl(std::get<TPosition>(lib.second), lib.first, TNodeFlags::ArbitraryContent));
+                if (const auto& first = std::get<1U>(lib.second)) {
+                    node = L(node, new TAstAtomNodeImpl(first->second, first->first, TNodeFlags::ArbitraryContent));
+                    if (const auto& second = std::get<2U>(lib.second)) {
+                        node = L(node, new TAstAtomNodeImpl(second->second, second->first, TNodeFlags::ArbitraryContent));
                     }
                 }
 
diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp
index c5261a1073e..8ca1cc3968e 100644
--- a/ydb/library/yql/sql/v1/sql.cpp
+++ b/ydb/library/yql/sql/v1/sql.cpp
@@ -10441,21 +10441,24 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
             }
 
             TString alias;
-            if (!values[0].GetLiteral(alias, Ctx)) {
+            if (!values.front().GetLiteral(alias, Ctx)) {
                 Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
                 return{};
             }
 
-            TMaybe<std::pair<TString, TString>> fileWithToken;
+            TContext::TLibraryStuff library;
+            std::get<TPosition>(library) = values.front().Build()->GetPos();
             if (values.size() > 1) {
-                fileWithToken.ConstructInPlace();
-                if (!values[1].GetLiteral(fileWithToken->first, Ctx)) {
+                auto& first = std::get<1U>(library);
+                first.emplace();
+                first->second = values[1].Build()->GetPos();
+                if (!values[1].GetLiteral(first->first, Ctx)) {
                     Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
                     return{};
                 }
 
                 TSet<TString> names;
-                SubstParameters(fileWithToken->first, Nothing(), &names);
+                SubstParameters(first->first, Nothing(), &names);
                 for (const auto& name : names) {
                     auto namedNode = GetNamedNode(name);
                     if (!namedNode) {
@@ -10463,14 +10466,17 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
                     }
                 }
                 if (values.size() > 2) {
-                    if (!values[2].GetLiteral(fileWithToken->second, Ctx)) {
+                    auto& second = std::get<2U>(library);
+                    second.emplace();
+                    second->second = values[2].Build()->GetPos();
+                    if (!values[2].GetLiteral(second->first, Ctx)) {
                         Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
                         return{};
                     }
                 }
             }
 
-            Ctx.Libraries[alias] = fileWithToken;
+            Ctx.Libraries[alias] = std::move(library);
             Ctx.IncrementMonCounter("sql_pragma", "library");
         } else if (normalizedPragma == "directread") {
             Ctx.PragmaDirectRead = true;