Add package pragma

102 files changed, 1367 insertions, 81 deletions

diff --git a/ydb/core/client/minikql_compile/yql_expr_minikql.cpp b/ydb/core/client/minikql_compile/yql_expr_minikql.cpp
index fff28f70b6..a32a38c6bf 100644
--- a/ydb/core/client/minikql_compile/yql_expr_minikql.cpp
+++ b/ydb/core/client/minikql_compile/yql_expr_minikql.cpp
@@ -1582,7 +1582,7 @@ private:
             return false;
         }
         Expr->Context.IssueManager.AddIssues(std::move(astRes.Issues));
-        if (!CompileExpr(*root, Expr->Root, Expr->Context, nullptr)) {
+        if (!CompileExpr(*root, Expr->Root, Expr->Context, nullptr, nullptr)) {
             errors = Expr->Context.IssueManager.GetIssues();
             return false;
         }
diff --git a/ydb/core/engine/minikql/flat_local_tx_minikql.h b/ydb/core/engine/minikql/flat_local_tx_minikql.h
index 8652b68e6a..1bb07bfd8b 100644
--- a/ydb/core/engine/minikql/flat_local_tx_minikql.h
+++ b/ydb/core/engine/minikql/flat_local_tx_minikql.h
@@ -95,7 +95,7 @@ class TFlatLocalMiniKQL : public NTabletFlatExecutor::ITransaction {
         }
         expr.Context.IssueManager.AddIssues(astResult.Issues);
 
-        if (!NYql::CompileExpr(*astResult.Root, expr.Root, expr.Context, nullptr)) {
+        if (!NYql::CompileExpr(*astResult.Root, expr.Root, expr.Context, nullptr, nullptr)) {
             errors = expr.Context.IssueManager.GetIssues();
             return false;
         }
diff --git a/ydb/core/kqp/host/kqp_host.cpp b/ydb/core/kqp/host/kqp_host.cpp
index 1ce9e72f6a..fd7a537d23 100644
--- a/ydb/core/kqp/host/kqp_host.cpp
+++ b/ydb/core/kqp/host/kqp_host.cpp
@@ -1144,7 +1144,7 @@ private:
         }
 
         TExprNode::TPtr result;
-        if (!CompileExpr(*astRes.Root, result, ctx, ModuleResolver.get())) {
+        if (!CompileExpr(*astRes.Root, result, ctx, ModuleResolver.get(), nullptr)) {
             return nullptr;
         }
 
diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp
index 80dfbd3d1c..82c0ff8e13 100644
--- a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp
+++ b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp
@@ -60,7 +60,7 @@ NYql::NNodes::TExprBase GetExpr(const TString& ast, NYql::TExprContext& ctx, NYq
     NYql::TAstParseResult astRes = NYql::ParseAst(ast);
     YQL_ENSURE(astRes.IsOk());
     NYql::TExprNode::TPtr result;
-    YQL_ENSURE(CompileExpr(*astRes.Root, result, ctx, moduleResolver));
+    YQL_ENSURE(CompileExpr(*astRes.Root, result, ctx, moduleResolver, nullptr));
     return NYql::NNodes::TExprBase(result);
 }
 
diff --git a/ydb/core/kqp/ut/query/kqp_query_ut.cpp b/ydb/core/kqp/ut/query/kqp_query_ut.cpp
index 0e262f53d2..18f0baf3e8 100644
--- a/ydb/core/kqp/ut/query/kqp_query_ut.cpp
+++ b/ydb/core/kqp/ut/query/kqp_query_ut.cpp
@@ -669,7 +669,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) {
         UNIT_ASSERT(astRes.IsOk());
         NYql::TExprContext exprCtx;
         NYql::TExprNode::TPtr exprRoot;
-        UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr));
+        UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr));
 
         UNIT_ASSERT(NJson::ValidateJson(result.GetPlan()));
     }
diff --git a/ydb/core/testlib/minikql_compile.h b/ydb/core/testlib/minikql_compile.h
index 4d76ca8d73..3c7f0b690e 100644
--- a/ydb/core/testlib/minikql_compile.h
+++ b/ydb/core/testlib/minikql_compile.h
@@ -82,7 +82,7 @@ inline TExprContainer::TPtr ParseText(const TString& programText) {
     UNIT_ASSERT(astRes.IsOk());
 
     TExprContainer::TPtr expr(new TExprContainer());
-    bool isOk = CompileExpr(*astRes.Root, expr->Root, expr->Context, nullptr);
+    bool isOk = CompileExpr(*astRes.Root, expr->Root, expr->Context, nullptr, nullptr);
     expr->Context.IssueManager.GetIssues().PrintTo(Cerr);
     UNIT_ASSERT(isOk);
     return expr;
diff --git a/ydb/docs/en/core/yql/reference/yql-core/syntax/_includes/pragma/files.md b/ydb/docs/en/core/yql/reference/yql-core/syntax/_includes/pragma/files.md
index 914c06c239..ac1d3e188e 100644
--- a/ydb/docs/en/core/yql/reference/yql-core/syntax/_includes/pragma/files.md
+++ b/ydb/docs/en/core/yql/reference/yql-core/syntax/_includes/pragma/files.md
@@ -56,5 +56,53 @@ IMPORT a SYMBOLS $x;
 SELECT $x;
 ```
 
-{% endif %}
+### Package
+
+| Value type | Default | Static /<br/>dynamic |
+| --- | --- | --- |
+| Two or three arguments: package name, URL and optional token | — | Static |
+
+Attach a hierarchical set of files to the query by URL, treating them as a package with a given name - an interconnected set of libraries.
+
+Package name is expected to be given as ``project_name.package_name``; from package's libraries you can do [IMPORT](../../export_import.md) with a module name like ``pkg.project_name.package_name.maybe.nested.module.name``.
+
+Example for a package with flat hierarchy which consists of two libraries - foo.sql and bar.sql:
+
+``` yql
+PRAGMA package("project.package", "{{ corporate-yt }}/{{ corporate-yt-cluster }}/path/to/package");
+IMPORT pkg.project.package.foo SYMBOLS $foo;
+IMPORT pkg.project.package.bar SYMBOLS $bar;
+SELECT $foo, $bar;
+```
+
+You can also use text parameter value substitution in the URL:
+
+``` yql
+DECLARE $_path AS STRING; -- "path"
+PRAGMA package("project.package","{{ corporate-yt }}/{{ corporate-yt-cluster }}/{$_path}/to/package");
+IMPORT pkg.project.package.foo SYMBOLS $foo;
+IMPORT pkg.project.package.bar SYMBOLS $bar;
+SELECT $foo, $bar;
+```
+
+### OverrideLibrary
 
+| Value type | Default | Static/<br/>dynamic |
+| --- | --- | --- |
+| One argument: the file name | — | Static |
+
+Treat the specified attached file as a library and override with it one of package's libraries.
+
+File name is expected to be given as ``project_name/package_name/maybe/nested/module/name.EXTENSION``, extensions analagous to [PRAGMA Library](#library) are supported.
+
+Example:
+
+```yql
+PRAGMA package("project.package", "{{ corporate-yt }}/{{ corporate-yt-cluster }}/path/to/package");
+PRAGMA override_library("project/package/maybe/nested/module/name.sql");
+
+IMPORT pkg.project.package.foo SYMBOLS $foo;
+SELECT $foo;
+```
+
+{% endif %}
diff --git a/ydb/docs/presets.yaml b/ydb/docs/presets.yaml
index 84cc777e0b..ea075190a3 100644
--- a/ydb/docs/presets.yaml
+++ b/ydb/docs/presets.yaml
@@ -42,3 +42,5 @@ default:
   ydb-doc-repo: https://github.com/ydb-platform/ydb/tree/main/ydb/docs
 
   corporate-paste: https://www.paste.example.com
+  corporate-yt: https://www.yt.example.com
+  corporate-yt-cluster: example-cluster
diff --git a/ydb/docs/ru/core/yql/reference/yql-core/syntax/_includes/pragma/files.md b/ydb/docs/ru/core/yql/reference/yql-core/syntax/_includes/pragma/files.md
index 64da39079b..e9be261dba 100644
--- a/ydb/docs/ru/core/yql/reference/yql-core/syntax/_includes/pragma/files.md
+++ b/ydb/docs/ru/core/yql/reference/yql-core/syntax/_includes/pragma/files.md
@@ -60,4 +60,53 @@ IMPORT a SYMBOLS $x;
 SELECT $x;
 ```
 
+### Package
+
+| Тип значения | По умолчанию | Статическая /<br/>динамическая |
+| --- | --- | --- |
+| Два или три аргумента - имя пакета, URL и опциональный токен | — | Статическая |
+
+Приложить иерархический набор файлов к запросу по URL, интерпретируя их в качестве пакета с указанным именем - взаимосвязанного набора библиотек.
+
+Имя пакета ожидается в формате ``project_name.package_name``; из библиотек пакета в дальнейшнем можно делать [IMPORT](../../export_import.md) с именем модуля вида ``pkg.project_name.package_name.maybe.nested.module.name``.
+
+Пример для пакета с плоской иерархией, состоящего из двух библиотек - foo.sql и bar.sql: 
+
+``` yql
+PRAGMA package("project.package", "{{ corporate-yt }}/{{ corporate-yt-cluster }}/path/to/package");
+IMPORT pkg.project.package.foo SYMBOLS $foo;
+IMPORT pkg.project.package.bar SYMBOLS $bar;
+SELECT $foo, $bar;
+```
+
+При этом можно использовать подстановку текстовых параметров в URL:
+
+``` yql
+DECLARE $_path AS STRING; -- "path"
+PRAGMA package("project.package","{{ corporate-yt }}/{{ corporate-yt-cluster }}/{$_path}/to/package");
+IMPORT pkg.project.package.foo SYMBOLS $foo;
+IMPORT pkg.project.package.bar SYMBOLS $bar;
+SELECT $foo, $bar;
+```
+
+### OverrideLibrary
+
+| Тип значения | По умолчанию | Статическая /<br/>динамическая |
+| --- | --- | --- |
+| Один аргумент - имя файла | — | Статическая |
+
+Интерпретировать указанный приложенный файл как библиотеку и перекрыть ей одну из библиотек пакета.
+
+Имя файла ожидается в формате ``project_name/package_name/maybe/nested/module/name.EXTENSION``, поддерживаются аналогичные [PRAGMA Library](#library) расширения.
+
+Пример:
+
+``` yql
+PRAGMA package("project.package", "{{ corporate-yt }}/{{ corporate-yt-cluster }}/path/to/package");
+PRAGMA override_library("project/package/maybe/nested/module/name.sql");
+
+IMPORT pkg.project.package.foo SYMBOLS $foo;
+SELECT $foo;
+```
+
 {% endif %}
diff --git a/ydb/library/yql/ast/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/ast/CMakeLists.darwin-x86_64.txt
index e8073842db..b486ffda55 100644
--- a/ydb/library/yql/ast/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/ast/CMakeLists.darwin-x86_64.txt
@@ -23,7 +23,9 @@ target_link_libraries(library-yql-ast PUBLIC
   cpp-yson-node
   yql-public-udf
   library-yql-utils
+  yql-utils-fetch
   yql-core-issue
+  core-url_lister-interface
   yql-parser-pg_catalog
 )
 target_sources(library-yql-ast PRIVATE
diff --git a/ydb/library/yql/ast/CMakeLists.linux-aarch64.txt b/ydb/library/yql/ast/CMakeLists.linux-aarch64.txt
index cca3dc9ca0..6a823d859e 100644
--- a/ydb/library/yql/ast/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/ast/CMakeLists.linux-aarch64.txt
@@ -24,7 +24,9 @@ target_link_libraries(library-yql-ast PUBLIC
   cpp-yson-node
   yql-public-udf
   library-yql-utils
+  yql-utils-fetch
   yql-core-issue
+  core-url_lister-interface
   yql-parser-pg_catalog
 )
 target_sources(library-yql-ast PRIVATE
diff --git a/ydb/library/yql/ast/CMakeLists.linux-x86_64.txt b/ydb/library/yql/ast/CMakeLists.linux-x86_64.txt
index cca3dc9ca0..6a823d859e 100644
--- a/ydb/library/yql/ast/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/ast/CMakeLists.linux-x86_64.txt
@@ -24,7 +24,9 @@ target_link_libraries(library-yql-ast PUBLIC
   cpp-yson-node
   yql-public-udf
   library-yql-utils
+  yql-utils-fetch
   yql-core-issue
+  core-url_lister-interface
   yql-parser-pg_catalog
 )
 target_sources(library-yql-ast PRIVATE
diff --git a/ydb/library/yql/ast/CMakeLists.windows-x86_64.txt b/ydb/library/yql/ast/CMakeLists.windows-x86_64.txt
index e8073842db..b486ffda55 100644
--- a/ydb/library/yql/ast/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/ast/CMakeLists.windows-x86_64.txt
@@ -23,7 +23,9 @@ target_link_libraries(library-yql-ast PUBLIC
   cpp-yson-node
   yql-public-udf
   library-yql-utils
+  yql-utils-fetch
   yql-core-issue
+  core-url_lister-interface
   yql-parser-pg_catalog
 )
 target_sources(library-yql-ast PRIVATE
diff --git a/ydb/library/yql/ast/ya.make b/ydb/library/yql/ast/ya.make
index d216bb0007..ea0cd2fd3e 100644
--- a/ydb/library/yql/ast/ya.make
+++ b/ydb/library/yql/ast/ya.make
@@ -34,7 +34,9 @@ PEERDIR(
     library/cpp/yson/node
     ydb/library/yql/public/udf
     ydb/library/yql/utils
+    ydb/library/yql/utils/fetch
     ydb/library/yql/core/issue
+    ydb/library/yql/core/url_lister/interface
     ydb/library/yql/parser/pg_catalog
 )
 
diff --git a/ydb/library/yql/ast/yql_expr.cpp b/ydb/library/yql/ast/yql_expr.cpp
index 12108ab769..c6970fe733 100644
--- a/ydb/library/yql/ast/yql_expr.cpp
+++ b/ydb/library/yql/ast/yql_expr.cpp
@@ -3,6 +3,7 @@
 #include "yql_gc_nodes.h"
 
 #include <ydb/library/yql/utils/utf8.h>
+#include <ydb/library/yql/utils/fetch/fetch.h>
 #include <ydb/library/yql/core/issue/yql_issue.h>
 
 #include <ydb/library/yql/parser/pg_catalog/catalog.h>
@@ -16,12 +17,17 @@
 #include <util/digest/murmur.h>
 #include <util/digest/city.h>
 #include <util/digest/numeric.h>
+#include <util/string/cast.h>
 
 #include <map>
+#include <unordered_set>
 
 namespace NYql {
 
 const TStringBuf ZeroString = "";
+const char Dot = '.';
+const char Sep = '/';
+const TStringBuf PkgPrefix = "pkg";
 
 void ReportError(TExprContext& ctx, const TIssue& issue) {
     ctx.AddError(issue);
@@ -114,11 +120,13 @@ namespace {
         TExprContext& Expr;
         TVector<TFrame> Frames;
         TLibraryCohesion Cohesion;
+        std::unordered_set<TString> OverrideLibraries;
 
         TNodeOnNodeOwnedMap DeepClones;
 
         const TAnnotationNodeMap* Annotations = nullptr;
         IModuleResolver* ModuleResolver = nullptr;
+        IUrlListerManager* UrlListerManager = nullptr;
         ui32 TypeAnnotationIndex = Max<ui32>();
         TString File;
         ui16 SyntaxVersion = 0;
@@ -891,6 +899,31 @@ namespace {
         return TAstNode::NewList(node->GetPosition(), children.data(), children.size(), pool);
     }
 
+    bool AddParameterDependencies(const TString& url, const TAstNode& node, TContext& ctx) {
+        auto world = ctx.FindBinding("world");
+        if (!world.empty()) {
+            TSet<TString> names;
+            SubstParameters(url, Nothing(), &names);
+            for (const auto& name : names) {
+                auto nameRef = ctx.FindBinding(name);
+                if (nameRef.empty()) {
+                    ctx.AddError(node, TStringBuilder() << "Name not found: " << name);
+                    return false;
+                }
+
+                TExprNode::TListType args = world;
+                args.insert(args.end(), nameRef.begin(), nameRef.end());
+                auto newWorld = TExprNode::TListType{ ctx.Expr.NewCallable(node.GetPosition(), "Left!", {
+                    ctx.Expr.NewCallable(node.GetPosition(), "Cons!", std::move(args)) })};
+
+                ctx.Frames.back().Bindings["world"] = newWorld;
+                world = newWorld;
+            }
+        }
+
+        return true;
+    }
+
     TExprNode::TListType Compile(const TAstNode& node, TContext& ctx);
 
     TExprNode::TPtr CompileQuote(const TAstNode& node, TContext& ctx) {
@@ -1210,27 +1243,8 @@ namespace {
             }
         }
 
-        if (url) {
-            auto world = ctx.FindBinding("world");
-            if (!world.empty()) {
-                TSet<TString> names;
-                SubstParameters(url, Nothing(), &names);
-                for (const auto& name : names) {
-                    auto nameRef = ctx.FindBinding(name);
-                    if (nameRef.empty()) {
-                        ctx.AddError(node, TStringBuilder() << "Name not found: " << name);
-                        return false;
-                    }
-
-                    TExprNode::TListType args = world;
-                    args.insert(args.end(), nameRef.begin(), nameRef.end());
-                    auto newWorld = TExprNode::TListType{ ctx.Expr.NewCallable(node.GetPosition(), "Left!", {
-                        ctx.Expr.NewCallable(node.GetPosition(), "Cons!", std::move(args)) })};
-
-                    ctx.Frames.back().Bindings["world"] = newWorld;
-                    world = newWorld;
-                }
-            }
+        if (url && !AddParameterDependencies(url, node, ctx)) {
+            return false;
         }
 
         if (!ctx.ModuleResolver) {
@@ -1250,6 +1264,149 @@ namespace {
         return true;
     }
 
+    bool CompilePackageDef(const TAstNode& node, TContext& ctx) {
+        if (node.GetChildrenCount() < 2 || node.GetChildrenCount() > 4) {
+            ctx.AddError(node, "Expected list of size from 2 to 4");
+            return false;
+        }
+
+        auto nameNode = node.GetChild(1);
+        if (!nameNode->IsAtom()) {
+            ctx.AddError(*nameNode, "Expected atom");
+            return false;
+        }
+
+        auto name = TString(nameNode->GetContent());
+
+        TString url;
+        if (node.GetChildrenCount() > 2) {
+            const auto file = node.GetChild(2);
+            if (!file->IsAtom()) {
+                ctx.AddError(*file, "Expected atom");
+                return false;
+            }
+
+            url = file->GetContent();
+        }
+
+        TString token;
+        if (node.GetChildrenCount() > 3) {
+            const auto tokenNode = node.GetChild(3);
+            if (!tokenNode->IsAtom()) {
+                ctx.AddError(*tokenNode, "Expected atom");
+                return false;
+            }
+
+            token = tokenNode->GetContent();
+        }
+
+        if (url && !AddParameterDependencies(url, node, ctx)) {
+            return false;
+        }
+
+        if (!ctx.ModuleResolver) {
+            return true;
+        }
+
+        if (!ctx.UrlListerManager) {
+            return true;
+        }
+
+        ctx.ModuleResolver->RegisterPackage(name);
+
+        auto packageModuleName = TStringBuilder() << PkgPrefix;
+
+        TStringBuf nameBuf(name);
+        while (auto part = nameBuf.NextTok(Dot)) {
+            packageModuleName << Sep << part;
+        }
+
+        auto queue = TVector<std::pair<TString, THttpURL>> {
+            {packageModuleName, ParseURL(url)}
+        };
+
+        while (queue) {
+            auto [prefix, httpUrl] = queue.back();
+            queue.pop_back();
+
+            TVector<TUrlListEntry> urlListEntries;
+            try {
+                urlListEntries = ctx.UrlListerManager->ListUrl(httpUrl, token);
+            } catch (const std::exception& e) {
+                ctx.AddError(*nameNode,
+                    TStringBuilder()
+                        << "UrlListerManager: failed to list URL \"" << httpUrl.PrintS()
+                        << "\", details: " << e.what()
+                );
+
+                return false;
+            }
+
+            for (auto& urlListEntry: urlListEntries) {
+                switch (urlListEntry.Type) {
+                case EUrlListEntryType::FILE: {
+                    auto moduleName = TStringBuilder()
+                        << prefix << Sep << urlListEntry.Name;
+
+                    if (ctx.OverrideLibraries.contains(moduleName)) {
+                        continue;
+                    }
+
+                    if (!ctx.ModuleResolver->AddFromUrl(
+                        moduleName, urlListEntry.Url.PrintS(), token, ctx.Expr,
+                        ctx.SyntaxVersion, 0, nameNode->GetPosition()
+                    )) {
+                        return false;
+                    }
+
+                    break;
+                }
+
+                case EUrlListEntryType::DIRECTORY: {
+                    queue.push_back({
+                        TStringBuilder() << prefix << Sep << urlListEntry.Name,
+                        urlListEntry.Url
+                    });
+
+                    break;
+                }
+                }
+            }
+        }
+
+        return true;
+    }
+
+    bool CompileOverrideLibraryDef(const TAstNode& node, TContext& ctx) {
+        if (node.GetChildrenCount() != 2) {
+            ctx.AddError(node, "Expected list of size 2");
+            return false;
+        }
+
+        auto nameNode = node.GetChild(1);
+        if (!nameNode->IsAtom()) {
+            ctx.AddError(*nameNode, "Expected atom");
+            return false;
+        }
+
+        if (!ctx.ModuleResolver) {
+            return true;
+        }
+
+        auto overrideLibraryName = TStringBuilder()
+            << PkgPrefix << Sep << nameNode->GetContent();
+
+        if (!ctx.ModuleResolver->AddFromFile(
+            overrideLibraryName, ctx.Expr, ctx.SyntaxVersion, 0, nameNode->GetPosition()
+        )) {
+            return false;
+        }
+
+        ctx.OverrideLibraries.insert(std::move(overrideLibraryName));
+
+        return true;
+    }
+
     bool CompileReturn(const TAstNode& node, TContext& ctx) {
         if (node.GetChildrenCount() < 2U) {
             ctx.AddError(node, "Expected non empty list.");
@@ -1309,6 +1466,14 @@ namespace {
                 } else if (firstChild->GetContent() == TStringBuf("declare")) {
                     if (!CompileDeclare(*node, ctx, false))
                         return {};
+                } else if (firstChild->GetContent() == TStringBuf("package")) {
+                    if (!CompilePackageDef(*node, ctx)) {
+                        return {};
+                    }
+                } else if (firstChild->GetContent() == TStringBuf("override_library")) {
+                    if (!CompileOverrideLibraryDef(*node, ctx)) {
+                        return {};
+                    }
                 }
             }
 
@@ -1378,6 +1543,16 @@ namespace {
                 }
 
                 continue;
+            } else if (firstChild->GetContent() == TStringBuf("package")) {
+                if (!topLevel) {
+                    ctx.AddError(*firstChild, "Package statements are only allowed on top level block");
+                    return {};
+                }
+            } else if (firstChild->GetContent() == TStringBuf("override_library")) {
+                if (!topLevel) {
+                    ctx.AddError(*firstChild, "override_library statements are only allowed on top level block");
+                    return {};
+                }
             } else {
                 ctx.AddError(*firstChild, ToString("expected either let, return or import, but have ") + firstChild->GetContent());
                 return {};
@@ -2070,7 +2245,8 @@ namespace {
 } // namespace
 
 bool CompileExpr(TAstNode& astRoot, TExprNode::TPtr& exprRoot, TExprContext& ctx,
-    IModuleResolver* resolver, bool hasAnnotations, ui32 typeAnnotationIndex, ui16 syntaxVersion) {
+    IModuleResolver* resolver, IUrlListerManager* urlListerManager,
+    bool hasAnnotations, ui32 typeAnnotationIndex, ui16 syntaxVersion) {
     exprRoot.Reset();
     TAstNode* cleanRoot = nullptr;
     TAnnotationNodeMap annotations;
@@ -2101,6 +2277,7 @@ bool CompileExpr(TAstNode& astRoot, TExprNode::TPtr& exprRoot, TExprContext& ctx
     compileCtx.Annotations = currentAnnotations;
     compileCtx.TypeAnnotationIndex = typeAnnotationIndex;
     compileCtx.ModuleResolver = resolver;
+    compileCtx.UrlListerManager = urlListerManager;
     compileCtx.PushFrame();
     auto world = compileCtx.Expr.NewWorld(astRoot.GetPosition());
     if (typeAnnotationIndex != Max<ui32>()) {
@@ -2117,7 +2294,8 @@ bool CompileExpr(TAstNode& astRoot, TExprNode::TPtr& exprRoot, TExprContext& ctx
 }
 
 bool CompileExpr(TAstNode& astRoot, TExprNode::TPtr& exprRoot, TExprContext& ctx,
-    IModuleResolver* resolver, ui32 annotationFlags, ui16 syntaxVersion)
+    IModuleResolver* resolver, IUrlListerManager* urlListerManager,
+    ui32 annotationFlags, ui16 syntaxVersion)
 {
     bool hasAnnotations = annotationFlags != TExprAnnotationFlags::None;
     ui32 typeAnnotationIndex = Max<ui32>();
@@ -2126,7 +2304,7 @@ bool CompileExpr(TAstNode& astRoot, TExprNode::TPtr& exprRoot, TExprContext& ctx
         typeAnnotationIndex = hasPostions ? 1 : 0;
     }
 
-    return CompileExpr(astRoot, exprRoot, ctx, resolver, hasAnnotations, typeAnnotationIndex, syntaxVersion);
+    return CompileExpr(astRoot, exprRoot, ctx, resolver, urlListerManager, hasAnnotations, typeAnnotationIndex, syntaxVersion);
 }
 
 bool CompileExpr(TAstNode& astRoot, TLibraryCohesion& library, TExprContext& ctx, ui16 syntaxVersion) {
diff --git a/ydb/library/yql/ast/yql_expr.h b/ydb/library/yql/ast/yql_expr.h
index d0f9a0e2b6..2babce4d7e 100644
--- a/ydb/library/yql/ast/yql_expr.h
+++ b/ydb/library/yql/ast/yql_expr.h
@@ -8,6 +8,7 @@
 #include "yql_constraint.h"
 #include "yql_pos_handle.h"
 
+#include <ydb/library/yql/core/url_lister/interface/url_lister_manager.h>
 #include <ydb/library/yql/utils/yql_panic.h>
 #include <ydb/library/yql/public/issue/yql_issue_manager.h>
 #include <ydb/library/yql/public/udf/udf_data_type.h>
@@ -2737,10 +2738,12 @@ private:
 };
 
 bool CompileExpr(TAstNode& astRoot, TExprNode::TPtr& exprRoot, TExprContext& ctx,
-    IModuleResolver* resolver, bool hasAnnotations = false, ui32 typeAnnotationIndex = Max<ui32>(), ui16 syntaxVersion = 0);
+    IModuleResolver* resolver, IUrlListerManager* urlListerManager,
+    bool hasAnnotations = false, ui32 typeAnnotationIndex = Max<ui32>(), ui16 syntaxVersion = 0);
 
 bool CompileExpr(TAstNode& astRoot, TExprNode::TPtr& exprRoot, TExprContext& ctx,
-    IModuleResolver* resolver, ui32 annotationFlags, ui16 syntaxVersion = 0);
+    IModuleResolver* resolver, IUrlListerManager* urlListerManager,
+    ui32 annotationFlags, ui16 syntaxVersion = 0);
 
 struct TLibraryCohesion {
     TExportTable Exports;
diff --git a/ydb/library/yql/ast/yql_expr_ut.cpp b/ydb/library/yql/ast/yql_expr_ut.cpp
index 06550f2507..0db8ecf506 100644
--- a/ydb/library/yql/ast/yql_expr_ut.cpp
+++ b/ydb/library/yql/ast/yql_expr_ut.cpp
@@ -15,7 +15,7 @@ Y_UNIT_TEST_SUITE(TCompileYqlExpr) {
     }
 
     static void CompileExprWithCheck(TAstNode& root, TExprNode::TPtr& exprRoot, TExprContext& exprCtx, ui32 typeAnnotationIndex = Max<ui32>()) {
-        const bool success = CompileExpr(root, exprRoot, exprCtx, nullptr, typeAnnotationIndex != Max<ui32>(), typeAnnotationIndex);
+        const bool success = CompileExpr(root, exprRoot, exprCtx, nullptr, nullptr, typeAnnotationIndex != Max<ui32>(), typeAnnotationIndex);
         exprCtx.IssueManager.GetIssues().PrintTo(Cout);
 
         UNIT_ASSERT(success);
@@ -33,7 +33,7 @@ Y_UNIT_TEST_SUITE(TCompileYqlExpr) {
         TAstParseResult astRes = ParseAstWithCheck(program);
         TExprContext exprCtx;
         TExprNode::TPtr exprRoot;
-        bool result = CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr);
+        bool result = CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr);
         exprCtx.IssueManager.GetIssues().PrintTo(Cout);
         return result;
     }
@@ -313,8 +313,8 @@ Y_UNIT_TEST_SUITE(TCompareExprTrees) {
         TExprContext ctxOne, ctxTwo;
         TExprNode::TPtr rootOne, rootTwo;
 
-        UNIT_ASSERT(CompileExpr(*progOne.Root, rootOne, ctxOne, nullptr));
-        UNIT_ASSERT(CompileExpr(*progTwo.Root, rootTwo, ctxTwo, nullptr));
+        UNIT_ASSERT(CompileExpr(*progOne.Root, rootOne, ctxOne, nullptr, nullptr));
+        UNIT_ASSERT(CompileExpr(*progTwo.Root, rootTwo, ctxTwo, nullptr, nullptr));
 
         const TExprNode* diffOne = rootOne.Get();
         const TExprNode* diffTwo = rootTwo.Get();
@@ -893,7 +893,7 @@ Y_UNIT_TEST_SUITE(TConvertToAst) {
         UNIT_ASSERT(astRes.IsOk());
         TExprContext exprCtx;
         TExprNode::TPtr exprRoot;
-        UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr));
+        UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr));
         UNIT_ASSERT(exprRoot);
 
         const auto convRes = ConvertToAst(*exprRoot, exprCtx, 0, true);
@@ -901,7 +901,7 @@ Y_UNIT_TEST_SUITE(TConvertToAst) {
 
         TExprContext exprCtx2;
         TExprNode::TPtr exprRoot2;
-        auto compileOk = CompileExpr(*convRes.Root, exprRoot2, exprCtx2, nullptr);
+        auto compileOk = CompileExpr(*convRes.Root, exprRoot2, exprCtx2, nullptr, nullptr);
         exprCtx2.IssueManager.GetIssues().PrintTo(Cout);
         UNIT_ASSERT(compileOk);
         UNIT_ASSERT(exprRoot2);
diff --git a/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt
index 6b6010bdea..e195fe28fd 100644
--- a/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt
@@ -20,6 +20,7 @@ add_subdirectory(peephole_opt)
 add_subdirectory(services)
 add_subdirectory(sql_types)
 add_subdirectory(type_ann)
+add_subdirectory(url_lister)
 add_subdirectory(url_preprocessing)
 add_subdirectory(user_data)
 add_subdirectory(ut)
@@ -65,6 +66,8 @@ target_link_libraries(library-yql-core PUBLIC
   yql-core-file_storage
   yql-core-sql_types
   yql-core-credentials
+  core-url_lister-interface
+  core-url_preprocessing-interface
   library-yql-minikql
   library-yql-protos
   yql-public-udf
diff --git a/ydb/library/yql/core/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/CMakeLists.linux-aarch64.txt
index 3d1fb5aec4..b4486180fb 100644
--- a/ydb/library/yql/core/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/core/CMakeLists.linux-aarch64.txt
@@ -20,6 +20,7 @@ add_subdirectory(peephole_opt)
 add_subdirectory(services)
 add_subdirectory(sql_types)
 add_subdirectory(type_ann)
+add_subdirectory(url_lister)
 add_subdirectory(url_preprocessing)
 add_subdirectory(user_data)
 add_subdirectory(ut)
@@ -66,6 +67,8 @@ target_link_libraries(library-yql-core PUBLIC
   yql-core-file_storage
   yql-core-sql_types
   yql-core-credentials
+  core-url_lister-interface
+  core-url_preprocessing-interface
   library-yql-minikql
   library-yql-protos
   yql-public-udf
diff --git a/ydb/library/yql/core/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/CMakeLists.linux-x86_64.txt
index 3d1fb5aec4..b4486180fb 100644
--- a/ydb/library/yql/core/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/core/CMakeLists.linux-x86_64.txt
@@ -20,6 +20,7 @@ add_subdirectory(peephole_opt)
 add_subdirectory(services)
 add_subdirectory(sql_types)
 add_subdirectory(type_ann)
+add_subdirectory(url_lister)
 add_subdirectory(url_preprocessing)
 add_subdirectory(user_data)
 add_subdirectory(ut)
@@ -66,6 +67,8 @@ target_link_libraries(library-yql-core PUBLIC
   yql-core-file_storage
   yql-core-sql_types
   yql-core-credentials
+  core-url_lister-interface
+  core-url_preprocessing-interface
   library-yql-minikql
   library-yql-protos
   yql-public-udf
diff --git a/ydb/library/yql/core/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/CMakeLists.windows-x86_64.txt
index 6b6010bdea..e195fe28fd 100644
--- a/ydb/library/yql/core/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/core/CMakeLists.windows-x86_64.txt
@@ -20,6 +20,7 @@ add_subdirectory(peephole_opt)
 add_subdirectory(services)
 add_subdirectory(sql_types)
 add_subdirectory(type_ann)
+add_subdirectory(url_lister)
 add_subdirectory(url_preprocessing)
 add_subdirectory(user_data)
 add_subdirectory(ut)
@@ -65,6 +66,8 @@ target_link_libraries(library-yql-core PUBLIC
   yql-core-file_storage
   yql-core-sql_types
   yql-core-credentials
+  core-url_lister-interface
+  core-url_preprocessing-interface
   library-yql-minikql
   library-yql-protos
   yql-public-udf
diff --git a/ydb/library/yql/core/facade/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/facade/CMakeLists.darwin-x86_64.txt
index 132297fce4..b013f0ae8a 100644
--- a/ydb/library/yql/core/facade/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/core/facade/CMakeLists.darwin-x86_64.txt
@@ -24,6 +24,8 @@ target_link_libraries(yql-core-facade PUBLIC
   yql-core-extract_predicate
   yql-core-file_storage
   yql-core-services
+  core-url_lister-interface
+  core-url_preprocessing-interface
   yql-core-credentials
   library-yql-sql
   yql-utils-log
diff --git a/ydb/library/yql/core/facade/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/facade/CMakeLists.linux-aarch64.txt
index f5858e8c2c..4a9a4ba75e 100644
--- a/ydb/library/yql/core/facade/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/core/facade/CMakeLists.linux-aarch64.txt
@@ -25,6 +25,8 @@ target_link_libraries(yql-core-facade PUBLIC
   yql-core-extract_predicate
   yql-core-file_storage
   yql-core-services
+  core-url_lister-interface
+  core-url_preprocessing-interface
   yql-core-credentials
   library-yql-sql
   yql-utils-log
diff --git a/ydb/library/yql/core/facade/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/facade/CMakeLists.linux-x86_64.txt
index f5858e8c2c..4a9a4ba75e 100644
--- a/ydb/library/yql/core/facade/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/core/facade/CMakeLists.linux-x86_64.txt
@@ -25,6 +25,8 @@ target_link_libraries(yql-core-facade PUBLIC
   yql-core-extract_predicate
   yql-core-file_storage
   yql-core-services
+  core-url_lister-interface
+  core-url_preprocessing-interface
   yql-core-credentials
   library-yql-sql
   yql-utils-log
diff --git a/ydb/library/yql/core/facade/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/facade/CMakeLists.windows-x86_64.txt
index 132297fce4..b013f0ae8a 100644
--- a/ydb/library/yql/core/facade/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/core/facade/CMakeLists.windows-x86_64.txt
@@ -24,6 +24,8 @@ target_link_libraries(yql-core-facade PUBLIC
   yql-core-extract_predicate
   yql-core-file_storage
   yql-core-services
+  core-url_lister-interface
+  core-url_preprocessing-interface
   yql-core-credentials
   library-yql-sql
   yql-utils-log
diff --git a/ydb/library/yql/core/facade/ya.make b/ydb/library/yql/core/facade/ya.make
index b605006573..82a2208b0a 100644
--- a/ydb/library/yql/core/facade/ya.make
+++ b/ydb/library/yql/core/facade/ya.make
@@ -15,6 +15,8 @@ PEERDIR(
     ydb/library/yql/core/extract_predicate
     ydb/library/yql/core/file_storage
     ydb/library/yql/core/services
+    ydb/library/yql/core/url_lister/interface
+    ydb/library/yql/core/url_preprocessing/interface
     ydb/library/yql/core/credentials
     ydb/library/yql/sql
     ydb/library/yql/utils/log
diff --git a/ydb/library/yql/core/facade/yql_facade.cpp b/ydb/library/yql/core/facade/yql_facade.cpp
index 6019a537d7..e17f908e9c 100644
--- a/ydb/library/yql/core/facade/yql_facade.cpp
+++ b/ydb/library/yql/core/facade/yql_facade.cpp
@@ -188,6 +188,10 @@ void TProgramFactory::SetArrowResolver(IArrowResolver::TPtr arrowResolver) {
     ArrowResolver_ = arrowResolver;
 }
 
+void TProgramFactory::SetUrlListerManager(IUrlListerManagerPtr urlListerManager) {
+    UrlListerManager_ = std::move(urlListerManager);
+}
+
 TProgramPtr TProgramFactory::Create(
         const TFile& file,
         const TString& sessionId)
@@ -210,11 +214,13 @@ TProgramPtr TProgramFactory::Create(
     TUdfIndex::TPtr udfIndex = UdfIndex_ ? UdfIndex_->Clone() : nullptr;
     TUdfIndexPackageSet::TPtr udfIndexPackageSet = (UdfIndexPackageSet_ && hiddenMode == EHiddenMode::Disable) ? UdfIndexPackageSet_->Clone() : nullptr;
     IModuleResolver::TPtr moduleResolver = Modules_ ? Modules_->CreateMutableChild() : nullptr;
+    IUrlListerManagerPtr urlListerManager = UrlListerManager_ ? UrlListerManager_->Clone() : nullptr;
     auto udfResolver = udfIndex ? NCommon::CreateUdfResolverWithIndex(udfIndex, UdfResolver_, FileStorage_) : UdfResolver_;
 
     // make UserDataTable_ copy here
     return new TProgram(FunctionRegistry_, randomProvider, timeProvider, NextUniqueId_, DataProvidersInit_,
-        UserDataTable_, Credentials_, moduleResolver, udfResolver, udfIndex, udfIndexPackageSet, FileStorage_, UrlPreprocessing_,
+        UserDataTable_, Credentials_, moduleResolver, urlListerManager,
+        udfResolver, udfIndex, udfIndexPackageSet, FileStorage_, UrlPreprocessing_,
         GatewaysConfig_, filename, sourceCode, sessionId, Runner_, EnableRangeComputeFor_, ArrowResolver_, hiddenMode);
 }
 
@@ -230,6 +236,7 @@ TProgram::TProgram(
         const TUserDataTable& userDataTable,
         const TCredentials::TPtr& credentials,
         const IModuleResolver::TPtr& modules,
+        const IUrlListerManagerPtr& urlListerManager,
         const IUdfResolver::TPtr& udfResolver,
         const TUdfIndex::TPtr& udfIndex,
         const TUdfIndexPackageSet::TPtr& udfIndexPackageSet,
@@ -250,6 +257,7 @@ TProgram::TProgram(
     , NextUniqueId_(nextUniqueId)
     , DataProvidersInit_(dataProvidersInit)
     , Credentials_(credentials)
+    , UrlListerManager_(urlListerManager)
     , UdfResolver_(udfResolver)
     , UdfIndex_(udfIndex)
     , UdfIndexPackageSet_(udfIndexPackageSet)
@@ -280,6 +288,12 @@ TProgram::TProgram(
         modules->SetUrlLoader(new TUrlLoader(FileStorage_));
         modules->SetCredentials(Credentials_);
     }
+
+    if (UrlListerManager_) {
+        UrlListerManager_->SetCredentials(Credentials_);
+        UrlListerManager_->SetUrlPreprocessing(urlPreprocessing);
+    }
+
     OperationOptions_.Runner = runner;
     UserDataStorage_->SetUrlPreprocessor(urlPreprocessing);
 }
@@ -353,6 +367,10 @@ void TProgram::SetParametersYson(const TString& parameters) {
     if (auto modules = dynamic_cast<TModuleResolver*>(Modules_.get())) {
         modules->SetParameters(node);
     }
+
+    if (UrlListerManager_) {
+        UrlListerManager_->SetParameters(node);
+    }
 }
 
 bool TProgram::ExtractQueryParametersMetadata() {
@@ -470,7 +488,11 @@ bool TProgram::Compile(const TString& username, bool skipLibraries) {
         }
     }
 
-    if (!CompileExpr(*AstRoot_, ExprRoot_, *ExprCtx_, skipLibraries ? nullptr : Modules_.get(), 0, SyntaxVersion_)) {
+    if (!CompileExpr(
+        *AstRoot_, ExprRoot_, *ExprCtx_,
+        skipLibraries ? nullptr : Modules_.get(),
+        skipLibraries ? nullptr : UrlListerManager_.Get(), 0, SyntaxVersion_
+    )) {
         return false;
     }
 
@@ -1437,6 +1459,7 @@ TTypeAnnotationContextPtr TProgram::BuildTypeAnnotationContext(const TString& us
     typeAnnotationContext->UserDataStorage = UserDataStorage_;
     typeAnnotationContext->Credentials = Credentials_;
     typeAnnotationContext->Modules = Modules_;
+    typeAnnotationContext->UrlListerManager = UrlListerManager_;
     typeAnnotationContext->UdfResolver = UdfResolver_;
     typeAnnotationContext->UdfIndex = UdfIndex_;
     typeAnnotationContext->UdfIndexPackageSet = UdfIndexPackageSet_;
diff --git a/ydb/library/yql/core/facade/yql_facade.h b/ydb/library/yql/core/facade/yql_facade.h
index 26529ed388..33ab474b85 100644
--- a/ydb/library/yql/core/facade/yql_facade.h
+++ b/ydb/library/yql/core/facade/yql_facade.h
@@ -4,6 +4,8 @@
 #include <ydb/library/yql/core/file_storage/file_storage.h>
 #include <ydb/library/yql/core/services/yql_plan.h>
 #include <ydb/library/yql/core/services/yql_transform_pipeline.h>
+#include <ydb/library/yql/core/url_lister/interface/url_lister_manager.h>
+#include <ydb/library/yql/core/url_preprocessing/interface/url_preprocessing.h>
 #include <ydb/library/yql/core/yql_type_annotation.h>
 #include <ydb/library/yql/core/yql_user_data.h>
 #include <ydb/library/yql/providers/config/yql_config_provider.h>
@@ -51,6 +53,7 @@ public:
     void SetCredentials(TCredentials::TPtr credentials);
     void SetGatewaysConfig(const TGatewaysConfig* gatewaysConfig);
     void SetModules(IModuleResolver::TPtr modules);
+    void SetUrlListerManager(IUrlListerManagerPtr urlListerManager);
     void SetUdfResolver(IUdfResolver::TPtr udfResolver);
     void SetUdfIndex(TUdfIndex::TPtr udfIndex, TUdfIndexPackageSet::TPtr udfIndexPackageSet);
     void SetFileStorage(TFileStoragePtr fileStorage);
@@ -79,6 +82,7 @@ private:
     TCredentials::TPtr Credentials_;
     const TGatewaysConfig* GatewaysConfig_;
     IModuleResolver::TPtr Modules_;
+    IUrlListerManagerPtr UrlListerManager_;
     IUdfResolver::TPtr UdfResolver_;
     TUdfIndex::TPtr UdfIndex_;
     TUdfIndexPackageSet::TPtr UdfIndexPackageSet_;
@@ -329,6 +333,7 @@ private:
         const TUserDataTable& userDataTable,
         const TCredentials::TPtr& credentials,
         const IModuleResolver::TPtr& modules,
+        const IUrlListerManagerPtr& urlListerManager,
         const IUdfResolver::TPtr& udfResolver,
         const TUdfIndex::TPtr& udfIndex,
         const TUdfIndexPackageSet::TPtr& udfIndexPackageSet,
@@ -375,6 +380,7 @@ private:
     TVector<TDataProviderInfo> DataProviders_;
     TYqlOperationOptions OperationOptions_;
     TCredentials::TPtr Credentials_;
+    const IUrlListerManagerPtr UrlListerManager_;
     const IUdfResolver::TPtr UdfResolver_;
     const TUdfIndex::TPtr UdfIndex_;
     const TUdfIndexPackageSet::TPtr UdfIndexPackageSet_;
diff --git a/ydb/library/yql/core/type_ann/type_ann_expr.cpp b/ydb/library/yql/core/type_ann/type_ann_expr.cpp
index 83f9b875a6..c9d433dc60 100644
--- a/ydb/library/yql/core/type_ann/type_ann_expr.cpp
+++ b/ydb/library/yql/core/type_ann/type_ann_expr.cpp
@@ -688,7 +688,7 @@ TExprNode::TPtr ParseAndAnnotate(
     }
 
     TExprNode::TPtr exprRoot;
-    if (!CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr)) {
+    if (!CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr)) {
         return nullptr;
     }
 
diff --git a/ydb/library/yql/core/type_ann/type_ann_types.cpp b/ydb/library/yql/core/type_ann/type_ann_types.cpp
index f596a2f6c9..3287fcb4eb 100644
--- a/ydb/library/yql/core/type_ann/type_ann_types.cpp
+++ b/ydb/library/yql/core/type_ann/type_ann_types.cpp
@@ -936,7 +936,7 @@ namespace NTypeAnnImpl {
             TAstNode::NewList(inputPos, pool,
                 TAstNode::NewLiteralAtom(inputPos, TStringBuf("return"), pool), parsedType));
         TExprNode::TPtr exprRoot;
-        if (!CompileExpr(*astRoot, exprRoot, ctx.Expr, nullptr)) {
+        if (!CompileExpr(*astRoot, exprRoot, ctx.Expr, nullptr, nullptr)) {
             return IGraphTransformer::TStatus::Error;
         }
 
diff --git a/ydb/library/yql/core/url_lister/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/url_lister/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..54bc4ee452
--- /dev/null
+++ b/ydb/library/yql/core/url_lister/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,21 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(interface)
+
+add_library(yql-core-url_lister)
+target_link_libraries(yql-core-url_lister PUBLIC
+  contrib-libs-cxxsupp
+  yutil
+  library-yql-ast
+  core-url_lister-interface
+  yql-utils-fetch
+)
+target_sources(yql-core-url_lister PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/url_lister/url_lister_manager.cpp
+)
diff --git a/ydb/library/yql/core/url_lister/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/url_lister/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..d330a86f00
--- /dev/null
+++ b/ydb/library/yql/core/url_lister/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,22 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(interface)
+
+add_library(yql-core-url_lister)
+target_link_libraries(yql-core-url_lister PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+  library-yql-ast
+  core-url_lister-interface
+  yql-utils-fetch
+)
+target_sources(yql-core-url_lister PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/url_lister/url_lister_manager.cpp
+)
diff --git a/ydb/library/yql/core/url_lister/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/url_lister/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..d330a86f00
--- /dev/null
+++ b/ydb/library/yql/core/url_lister/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,22 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(interface)
+
+add_library(yql-core-url_lister)
+target_link_libraries(yql-core-url_lister PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+  library-yql-ast
+  core-url_lister-interface
+  yql-utils-fetch
+)
+target_sources(yql-core-url_lister PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/url_lister/url_lister_manager.cpp
+)
diff --git a/ydb/library/yql/core/url_lister/CMakeLists.txt b/ydb/library/yql/core/url_lister/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/ydb/library/yql/core/url_lister/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+  include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+  include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+  include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+  include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/core/url_lister/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/url_lister/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..54bc4ee452
--- /dev/null
+++ b/ydb/library/yql/core/url_lister/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,21 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(interface)
+
+add_library(yql-core-url_lister)
+target_link_libraries(yql-core-url_lister PUBLIC
+  contrib-libs-cxxsupp
+  yutil
+  library-yql-ast
+  core-url_lister-interface
+  yql-utils-fetch
+)
+target_sources(yql-core-url_lister PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/url_lister/url_lister_manager.cpp
+)
diff --git a/ydb/library/yql/core/url_lister/interface/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/url_lister/interface/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..8f5f2f362a
--- /dev/null
+++ b/ydb/library/yql/core/url_lister/interface/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,22 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-url_lister-interface)
+target_link_libraries(core-url_lister-interface PUBLIC
+  contrib-libs-cxxsupp
+  yutil
+  library-cpp-uri
+  cpp-yson-node
+  yql-core-credentials
+  core-url_preprocessing-interface
+)
+target_sources(core-url_lister-interface PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/url_lister/interface/url_lister.cpp
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/url_lister/interface/url_lister_manager.cpp
+)
diff --git a/ydb/library/yql/core/url_lister/interface/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/url_lister/interface/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..3dd8ff2f4d
--- /dev/null
+++ b/ydb/library/yql/core/url_lister/interface/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,23 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-url_lister-interface)
+target_link_libraries(core-url_lister-interface PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+  library-cpp-uri
+  cpp-yson-node
+  yql-core-credentials
+  core-url_preprocessing-interface
+)
+target_sources(core-url_lister-interface PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/url_lister/interface/url_lister.cpp
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/url_lister/interface/url_lister_manager.cpp
+)
diff --git a/ydb/library/yql/core/url_lister/interface/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/url_lister/interface/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..3dd8ff2f4d
--- /dev/null
+++ b/ydb/library/yql/core/url_lister/interface/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,23 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-url_lister-interface)
+target_link_libraries(core-url_lister-interface PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+  library-cpp-uri
+  cpp-yson-node
+  yql-core-credentials
+  core-url_preprocessing-interface
+)
+target_sources(core-url_lister-interface PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/url_lister/interface/url_lister.cpp
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/url_lister/interface/url_lister_manager.cpp
+)
diff --git a/ydb/library/yql/core/url_lister/interface/CMakeLists.txt b/ydb/library/yql/core/url_lister/interface/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/ydb/library/yql/core/url_lister/interface/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+  include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+  include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+  include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+  include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/core/url_lister/interface/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/url_lister/interface/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..8f5f2f362a
--- /dev/null
+++ b/ydb/library/yql/core/url_lister/interface/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,22 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-url_lister-interface)
+target_link_libraries(core-url_lister-interface PUBLIC
+  contrib-libs-cxxsupp
+  yutil
+  library-cpp-uri
+  cpp-yson-node
+  yql-core-credentials
+  core-url_preprocessing-interface
+)
+target_sources(core-url_lister-interface PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/url_lister/interface/url_lister.cpp
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/url_lister/interface/url_lister_manager.cpp
+)
diff --git a/ydb/library/yql/core/url_lister/interface/url_lister.cpp b/ydb/library/yql/core/url_lister/interface/url_lister.cpp
new file mode 100644
index 0000000000..cd609cd07f
--- /dev/null
+++ b/ydb/library/yql/core/url_lister/interface/url_lister.cpp
@@ -0,0 +1 @@
+#include "url_lister.h"
diff --git a/ydb/library/yql/core/url_lister/interface/url_lister.h b/ydb/library/yql/core/url_lister/interface/url_lister.h
new file mode 100644
index 0000000000..4b7acd00ac
--- /dev/null
+++ b/ydb/library/yql/core/url_lister/interface/url_lister.h
@@ -0,0 +1,32 @@
+#pragma once
+
+#include <library/cpp/uri/http_url.h>
+
+#include <util/generic/ptr.h>
+#include <util/generic/string.h>
+#include <util/generic/vector.h>
+
+
+namespace NYql {
+
+enum class EUrlListEntryType {
+    FILE = 1,
+    DIRECTORY = 2,
+};
+
+
+struct TUrlListEntry {
+    THttpURL Url;
+    TString Name;
+    EUrlListEntryType Type;
+};
+
+
+class IUrlLister : public TThrRefBase {
+public:
+    virtual bool Accept(const THttpURL& url) const = 0;
+    virtual TVector<TUrlListEntry> ListUrl(const THttpURL& url, const TString& token) const = 0;
+};
+using IUrlListerPtr = TIntrusivePtr<IUrlLister>;
+
+}
diff --git a/ydb/library/yql/core/url_lister/interface/url_lister_manager.cpp b/ydb/library/yql/core/url_lister/interface/url_lister_manager.cpp
new file mode 100644
index 0000000000..7ccdbdb299
--- /dev/null
+++ b/ydb/library/yql/core/url_lister/interface/url_lister_manager.cpp
@@ -0,0 +1 @@
+#include "url_lister_manager.h"
diff --git a/ydb/library/yql/core/url_lister/interface/url_lister_manager.h b/ydb/library/yql/core/url_lister/interface/url_lister_manager.h
new file mode 100644
index 0000000000..910c4d36a1
--- /dev/null
+++ b/ydb/library/yql/core/url_lister/interface/url_lister_manager.h
@@ -0,0 +1,29 @@
+#pragma once
+
+#include "url_lister.h"
+
+#include <library/cpp/yson/node/node.h>
+
+#include <ydb/library/yql/core/credentials/yql_credentials.h>
+#include <ydb/library/yql/core/url_preprocessing/interface/url_preprocessing.h>
+
+#include <util/generic/ptr.h>
+
+
+namespace NYql {
+
+class IUrlListerManager: public TThrRefBase {
+public:
+    virtual TVector<TUrlListEntry> ListUrl(const THttpURL& url, const TString& tokenName) const = 0;
+
+public:
+    virtual TIntrusivePtr<IUrlListerManager> Clone() const = 0;
+
+    virtual void SetCredentials(TCredentials::TPtr credentials) = 0;
+    virtual void SetUrlPreprocessing(IUrlPreprocessing::TPtr urlPreprocessing) = 0;
+    virtual void SetParameters(const NYT::TNode& parameters) = 0;
+};
+
+using IUrlListerManagerPtr = TIntrusivePtr<IUrlListerManager>;
+
+}
diff --git a/ydb/library/yql/core/url_lister/interface/ya.make b/ydb/library/yql/core/url_lister/interface/ya.make
new file mode 100644
index 0000000000..524568ef10
--- /dev/null
+++ b/ydb/library/yql/core/url_lister/interface/ya.make
@@ -0,0 +1,15 @@
+LIBRARY()
+
+SRCS(
+    url_lister.cpp
+    url_lister_manager.cpp
+)
+
+PEERDIR(
+    library/cpp/uri
+    library/cpp/yson/node
+    ydb/library/yql/core/credentials
+    ydb/library/yql/core/url_preprocessing/interface
+)
+
+END()
diff --git a/ydb/library/yql/core/url_lister/url_lister_manager.cpp b/ydb/library/yql/core/url_lister/url_lister_manager.cpp
new file mode 100644
index 0000000000..69a4f98f06
--- /dev/null
+++ b/ydb/library/yql/core/url_lister/url_lister_manager.cpp
@@ -0,0 +1,96 @@
+#include "url_lister_manager.h"
+
+#include <ydb/library/yql/ast/yql_expr.h>
+#include <ydb/library/yql/utils/fetch/fetch.h>
+
+
+namespace NYql::NPrivate {
+
+class TUrlListerManager: public IUrlListerManager {
+public:
+    TUrlListerManager(
+        TVector<IUrlListerPtr> urlListers
+    )
+        : UrlListers(std::move(urlListers))
+    {
+    }
+
+public:
+    TVector<TUrlListEntry> ListUrl(const THttpURL& url, const TString& tokenName) const override {
+        auto urlString = SubstParameters(url.PrintS(), Parameters, nullptr);
+
+        if (UrlPreprocessing) {
+            auto [preprocessedUrlString, alias] = UrlPreprocessing->Preprocess(urlString);
+            urlString = preprocessedUrlString;
+        }
+
+        auto patchedUrl = ParseURL(urlString);
+
+        TString token;
+        if (tokenName) {
+            if (!Credentials) {
+                ythrow yexception() << "Missing credentials";
+            }
+
+            auto credential = Credentials->FindCredential(tokenName);
+            if (!credential) {
+                ythrow yexception() << "Unknown token name: " << tokenName;
+            }
+
+            token = credential->Content;
+        }
+
+        for (const auto& urlLister: UrlListers) {
+            if (urlLister->Accept(patchedUrl)) {
+                return urlLister->ListUrl(patchedUrl, token);
+            }
+        }
+
+        ythrow yexception() << "Unsupported package url: " << urlString;
+    }
+
+public:
+    IUrlListerManagerPtr Clone() const override {
+        auto clone = MakeUrlListerManager(UrlListers);
+
+        clone->SetCredentials(Credentials);
+        clone->SetUrlPreprocessing(UrlPreprocessing);
+
+        if (Parameters) {
+            clone->SetParameters(*Parameters);
+        }
+
+        return clone;
+    }
+
+    void SetCredentials(TCredentials::TPtr credentials) override {
+        Credentials = std::move(credentials);
+    }
+
+    void SetUrlPreprocessing(IUrlPreprocessing::TPtr urlPreprocessing) override {
+        UrlPreprocessing = std::move(urlPreprocessing);
+    }
+
+    void SetParameters(const NYT::TNode& parameters) override {
+        Parameters = parameters;
+    }
+
+private:
+    TVector<IUrlListerPtr> UrlListers;
+
+    TCredentials::TPtr Credentials;
+    IUrlPreprocessing::TPtr UrlPreprocessing;
+    TMaybe<NYT::TNode> Parameters;
+};
+
+}
+
+namespace NYql {
+
+IUrlListerManagerPtr MakeUrlListerManager(
+    TVector<IUrlListerPtr> urlListers
+) {
+    return MakeIntrusive<NPrivate::TUrlListerManager>(std::move(urlListers));
+}
+
+}
diff --git a/ydb/library/yql/core/url_lister/url_lister_manager.h b/ydb/library/yql/core/url_lister/url_lister_manager.h
new file mode 100644
index 0000000000..7b9061b79f
--- /dev/null
+++ b/ydb/library/yql/core/url_lister/url_lister_manager.h
@@ -0,0 +1,16 @@
+#pragma once
+
+#include <ydb/library/yql/core/url_lister/interface/url_lister.h>
+#include <ydb/library/yql/core/url_lister/interface/url_lister_manager.h>
+
+#include <util/generic/ptr.h>
+#include <util/generic/vector.h>
+
+
+namespace NYql {
+
+IUrlListerManagerPtr MakeUrlListerManager(
+    TVector<IUrlListerPtr> urlListers
+);
+
+}
diff --git a/ydb/library/yql/core/url_lister/ya.make b/ydb/library/yql/core/url_lister/ya.make
new file mode 100644
index 0000000000..2caf6cf6f0
--- /dev/null
+++ b/ydb/library/yql/core/url_lister/ya.make
@@ -0,0 +1,13 @@
+LIBRARY()
+
+SRCS(
+    url_lister_manager.cpp
+)
+
+PEERDIR(
+    ydb/library/yql/ast
+    ydb/library/yql/core/url_lister/interface
+    ydb/library/yql/utils/fetch
+)
+
+END()
diff --git a/ydb/library/yql/core/url_preprocessing/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/url_preprocessing/CMakeLists.darwin-x86_64.txt
index e94218ed23..f88c1b8fef 100644
--- a/ydb/library/yql/core/url_preprocessing/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/core/url_preprocessing/CMakeLists.darwin-x86_64.txt
@@ -6,11 +6,13 @@
 # original buildsystem will not be accepted.
 
 
+add_subdirectory(interface)
 
 add_library(yql-core-url_preprocessing)
 target_link_libraries(yql-core-url_preprocessing PUBLIC
   contrib-libs-cxxsupp
   yutil
+  core-url_preprocessing-interface
   providers-common-proto
   yql-utils-log
   cpp-regex-pcre
diff --git a/ydb/library/yql/core/url_preprocessing/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/url_preprocessing/CMakeLists.linux-aarch64.txt
index 8f61fc1505..2f92089e03 100644
--- a/ydb/library/yql/core/url_preprocessing/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/core/url_preprocessing/CMakeLists.linux-aarch64.txt
@@ -6,12 +6,14 @@
 # original buildsystem will not be accepted.
 
 
+add_subdirectory(interface)
 
 add_library(yql-core-url_preprocessing)
 target_link_libraries(yql-core-url_preprocessing PUBLIC
   contrib-libs-linux-headers
   contrib-libs-cxxsupp
   yutil
+  core-url_preprocessing-interface
   providers-common-proto
   yql-utils-log
   cpp-regex-pcre
diff --git a/ydb/library/yql/core/url_preprocessing/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/url_preprocessing/CMakeLists.linux-x86_64.txt
index 8f61fc1505..2f92089e03 100644
--- a/ydb/library/yql/core/url_preprocessing/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/core/url_preprocessing/CMakeLists.linux-x86_64.txt
@@ -6,12 +6,14 @@
 # original buildsystem will not be accepted.
 
 
+add_subdirectory(interface)
 
 add_library(yql-core-url_preprocessing)
 target_link_libraries(yql-core-url_preprocessing PUBLIC
   contrib-libs-linux-headers
   contrib-libs-cxxsupp
   yutil
+  core-url_preprocessing-interface
   providers-common-proto
   yql-utils-log
   cpp-regex-pcre
diff --git a/ydb/library/yql/core/url_preprocessing/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/url_preprocessing/CMakeLists.windows-x86_64.txt
index e94218ed23..f88c1b8fef 100644
--- a/ydb/library/yql/core/url_preprocessing/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/core/url_preprocessing/CMakeLists.windows-x86_64.txt
@@ -6,11 +6,13 @@
 # original buildsystem will not be accepted.
 
 
+add_subdirectory(interface)
 
 add_library(yql-core-url_preprocessing)
 target_link_libraries(yql-core-url_preprocessing PUBLIC
   contrib-libs-cxxsupp
   yutil
+  core-url_preprocessing-interface
   providers-common-proto
   yql-utils-log
   cpp-regex-pcre
diff --git a/ydb/library/yql/core/url_preprocessing/interface/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/url_preprocessing/interface/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..e1f0a03632
--- /dev/null
+++ b/ydb/library/yql/core/url_preprocessing/interface/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-url_preprocessing-interface)
+target_link_libraries(core-url_preprocessing-interface PUBLIC
+  contrib-libs-cxxsupp
+  yutil
+)
+target_sources(core-url_preprocessing-interface PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/url_preprocessing/interface/url_preprocessing.cpp
+)
diff --git a/ydb/library/yql/core/url_preprocessing/interface/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/url_preprocessing/interface/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..d3517baff9
--- /dev/null
+++ b/ydb/library/yql/core/url_preprocessing/interface/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,18 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-url_preprocessing-interface)
+target_link_libraries(core-url_preprocessing-interface PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+)
+target_sources(core-url_preprocessing-interface PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/url_preprocessing/interface/url_preprocessing.cpp
+)
diff --git a/ydb/library/yql/core/url_preprocessing/interface/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/url_preprocessing/interface/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..d3517baff9
--- /dev/null
+++ b/ydb/library/yql/core/url_preprocessing/interface/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,18 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-url_preprocessing-interface)
+target_link_libraries(core-url_preprocessing-interface PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+)
+target_sources(core-url_preprocessing-interface PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/url_preprocessing/interface/url_preprocessing.cpp
+)
diff --git a/ydb/library/yql/core/url_preprocessing/interface/CMakeLists.txt b/ydb/library/yql/core/url_preprocessing/interface/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/ydb/library/yql/core/url_preprocessing/interface/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+  include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+  include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+  include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+  include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/core/url_preprocessing/interface/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/url_preprocessing/interface/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..e1f0a03632
--- /dev/null
+++ b/ydb/library/yql/core/url_preprocessing/interface/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-url_preprocessing-interface)
+target_link_libraries(core-url_preprocessing-interface PUBLIC
+  contrib-libs-cxxsupp
+  yutil
+)
+target_sources(core-url_preprocessing-interface PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/url_preprocessing/interface/url_preprocessing.cpp
+)
diff --git a/ydb/library/yql/core/url_preprocessing/interface/url_preprocessing.cpp b/ydb/library/yql/core/url_preprocessing/interface/url_preprocessing.cpp
new file mode 100644
index 0000000000..ab51aea88b
--- /dev/null
+++ b/ydb/library/yql/core/url_preprocessing/interface/url_preprocessing.cpp
@@ -0,0 +1 @@
+#include "url_preprocessing.h"
diff --git a/ydb/library/yql/core/url_preprocessing/interface/url_preprocessing.h b/ydb/library/yql/core/url_preprocessing/interface/url_preprocessing.h
new file mode 100644
index 0000000000..6164f522f1
--- /dev/null
+++ b/ydb/library/yql/core/url_preprocessing/interface/url_preprocessing.h
@@ -0,0 +1,18 @@
+#pragma once
+
+#include <util/generic/ptr.h>
+#include <util/generic/string.h>
+
+#include <utility>
+
+
+namespace NYql {
+
+struct IUrlPreprocessing: public TThrRefBase {
+public:
+    using TPtr = TIntrusivePtr<IUrlPreprocessing>;
+    // Returns pair of <new url>, <url alias>
+    virtual std::pair<TString, TString> Preprocess(const TString& url) = 0;
+};
+
+}
diff --git a/ydb/library/yql/core/url_preprocessing/interface/ya.make b/ydb/library/yql/core/url_preprocessing/interface/ya.make
new file mode 100644
index 0000000000..26c9fe6bc9
--- /dev/null
+++ b/ydb/library/yql/core/url_preprocessing/interface/ya.make
@@ -0,0 +1,7 @@
+LIBRARY()
+
+SRCS(
+    url_preprocessing.cpp
+)
+
+END()
diff --git a/ydb/library/yql/core/url_preprocessing/url_preprocessing.cpp b/ydb/library/yql/core/url_preprocessing/url_preprocessing.cpp
index 1be039704f..2304cf2f1f 100644
--- a/ydb/library/yql/core/url_preprocessing/url_preprocessing.cpp
+++ b/ydb/library/yql/core/url_preprocessing/url_preprocessing.cpp
@@ -45,8 +45,8 @@ std::pair<TString, TString> TUrlPreprocessing::Preprocess(const TString& url) {
         if (auto a = AllowedUrls_.Match(convertedUrl)) {
             alias = *a;
         } else {
-            YQL_LOG(WARN) << "Url " << convertedUrl << " is not in allowed list, reject downloading";
-            ythrow yexception() << "It is not allowed to download url " << url;
+            YQL_LOG(WARN) << "Url " << convertedUrl << " is not in allowed list, reject accessing";
+            ythrow yexception() << "It is not allowed to access url " << url;
         }
     }
     YQL_LOG(INFO) << "UrlPreprocessing: " << convertedUrl << ", alias=" << alias;
diff --git a/ydb/library/yql/core/url_preprocessing/url_preprocessing.h b/ydb/library/yql/core/url_preprocessing/url_preprocessing.h
index 8877dd93a8..a01c0a7fa3 100644
--- a/ydb/library/yql/core/url_preprocessing/url_preprocessing.h
+++ b/ydb/library/yql/core/url_preprocessing/url_preprocessing.h
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <ydb/library/yql/core/url_preprocessing/interface/url_preprocessing.h>
 #include <ydb/library/yql/core/url_preprocessing/pattern_group.h>
 #include <ydb/library/yql/core/url_preprocessing/url_mapper.h>
 
diff --git a/ydb/library/yql/core/url_preprocessing/ya.make b/ydb/library/yql/core/url_preprocessing/ya.make
index de914ed783..c062ee36f2 100644
--- a/ydb/library/yql/core/url_preprocessing/ya.make
+++ b/ydb/library/yql/core/url_preprocessing/ya.make
@@ -7,10 +7,10 @@ SRCS(
 )
 
 PEERDIR(
+    ydb/library/yql/core/url_preprocessing/interface
     ydb/library/yql/providers/common/proto
     ydb/library/yql/utils/log
     library/cpp/regex/pcre
 )
 
 END()
-
diff --git a/ydb/library/yql/core/ya.make b/ydb/library/yql/core/ya.make
index cf865d2d8f..1dbac92b4c 100644
--- a/ydb/library/yql/core/ya.make
+++ b/ydb/library/yql/core/ya.make
@@ -64,6 +64,8 @@ PEERDIR(
     ydb/library/yql/core/file_storage
     ydb/library/yql/core/sql_types
     ydb/library/yql/core/credentials
+    ydb/library/yql/core/url_lister/interface
+    ydb/library/yql/core/url_preprocessing/interface
     ydb/library/yql/minikql
     ydb/library/yql/protos
     ydb/library/yql/public/udf
diff --git a/ydb/library/yql/core/yql_type_annotation.cpp b/ydb/library/yql/core/yql_type_annotation.cpp
index c1093b0986..1c63948080 100644
--- a/ydb/library/yql/core/yql_type_annotation.cpp
+++ b/ydb/library/yql/core/yql_type_annotation.cpp
@@ -214,6 +214,7 @@ bool TModuleResolver::AddFromUrl(const std::string_view& file, const std::string
         }
         block.UrlToken = cred->Content;
     }
+
     UserData->AddUserDataBlock(file, block);
 
     return AddFromFile(file, ctx, syntaxVersion, packageVersion, pos);
diff --git a/ydb/library/yql/core/yql_type_annotation.h b/ydb/library/yql/core/yql_type_annotation.h
index be99440e41..f3f5e595f3 100644
--- a/ydb/library/yql/core/yql_type_annotation.h
+++ b/ydb/library/yql/core/yql_type_annotation.h
@@ -8,6 +8,7 @@
 
 #include <ydb/library/yql/public/udf/udf_validate.h>
 #include <ydb/library/yql/core/credentials/yql_credentials.h>
+#include <ydb/library/yql/core/url_lister/interface/url_lister_manager.h>
 #include <ydb/library/yql/ast/yql_expr.h>
 
 #include <library/cpp/yson/node/node.h>
@@ -202,6 +203,7 @@ struct TTypeAnnotationContext: public TThrRefBase {
     TYqlOperationOptions OperationOptions;
     TCredentials::TPtr Credentials = MakeIntrusive<TCredentials>();
     IModuleResolver::TPtr Modules;
+    IUrlListerManagerPtr UrlListerManager;
     NUdf::EValidateMode ValidateMode = NUdf::EValidateMode::None;
     bool DisableNativeUdfSupport = false;
     TMaybe<TString> OptLLVM;
diff --git a/ydb/library/yql/core/yql_user_data.h b/ydb/library/yql/core/yql_user_data.h
index 97877b2990..c7f9cfd078 100644
--- a/ydb/library/yql/core/yql_user_data.h
+++ b/ydb/library/yql/core/yql_user_data.h
@@ -106,11 +106,4 @@ using TUserDataTable = THashMap<TUserDataKey, TUserDataBlock, TUserDataKey::THas
 
 using TTokenResolver = std::function<TString(const TString&, const TString&)>;
 
-struct IUrlPreprocessing: public TThrRefBase {
-public:
-    using TPtr = TIntrusivePtr<IUrlPreprocessing>;
-    // Returns pair of <new url>, <url alias>
-    virtual std::pair<TString, TString> Preprocess(const TString& url) = 0;
-};
-
 } // namespace NYql
diff --git a/ydb/library/yql/core/yql_user_data_storage.h b/ydb/library/yql/core/yql_user_data_storage.h
index ef503e071c..8db9deac71 100644
--- a/ydb/library/yql/core/yql_user_data_storage.h
+++ b/ydb/library/yql/core/yql_user_data_storage.h
@@ -5,6 +5,7 @@
 #include "yql_udf_resolver.h"
 #include "yql_user_data.h"
 
+#include <ydb/library/yql/core/url_preprocessing/interface/url_preprocessing.h>
 #include <ydb/library/yql/protos/yql_mount.pb.h>
 
 #include <util/generic/maybe.h>
diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_load_meta.cpp b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_load_meta.cpp
index e953cb3847..67ae01c1c3 100644
--- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_load_meta.cpp
+++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_load_meta.cpp
@@ -262,7 +262,7 @@ private:
                 const auto astRoot = TAstNode::NewList({}, pool,
                     TAstNode::NewList({}, pool, TAstNode::NewLiteralAtom({}, TStringBuf("return"), pool), parsedType));
                 TExprNode::TPtr exprRoot;
-                YQL_ENSURE(CompileExpr(*astRoot, exprRoot, ctx, nullptr), "Failed to compile.");
+                YQL_ENSURE(CompileExpr(*astRoot, exprRoot, ctx, nullptr, nullptr), "Failed to compile.");
 
                 // TODO: Collect type annotation directly from AST.
                 const auto callableTransformer = CreateExtCallableTypeAnnotationTransformer(*State_->Types);
diff --git a/ydb/library/yql/providers/common/comp_nodes/yql_parsetypehandle.cpp b/ydb/library/yql/providers/common/comp_nodes/yql_parsetypehandle.cpp
index 8919e250a1..fb46bcd990 100644
--- a/ydb/library/yql/providers/common/comp_nodes/yql_parsetypehandle.cpp
+++ b/ydb/library/yql/providers/common/comp_nodes/yql_parsetypehandle.cpp
@@ -37,7 +37,7 @@ public:
             NYql::TAstNode::NewList({}, pool,
                 NYql::TAstNode::NewLiteralAtom({}, TStringBuf("return"), pool), parsedType));
         NYql::TExprNode::TPtr exprRoot;
-        if (!CompileExpr(*astRoot, exprRoot, *exprCtxPtr, nullptr)) {
+        if (!CompileExpr(*astRoot, exprRoot, *exprCtxPtr, nullptr, nullptr)) {
             UdfTerminate(exprCtxPtr->IssueManager.GetIssues().ToString().data());
         }
 
diff --git a/ydb/library/yql/providers/yt/lib/CMakeLists.txt b/ydb/library/yql/providers/yt/lib/CMakeLists.txt
index 862a7b1c34..b4ad8e0282 100644
--- a/ydb/library/yql/providers/yt/lib/CMakeLists.txt
+++ b/ydb/library/yql/providers/yt/lib/CMakeLists.txt
@@ -23,3 +23,4 @@ add_subdirectory(skiff)
 add_subdirectory(url_mapper)
 add_subdirectory(yson_helpers)
 add_subdirectory(yt_download)
+add_subdirectory(yt_url_lister)
diff --git a/ydb/library/yql/providers/yt/lib/yt_url_lister/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/yt/lib/yt_url_lister/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..83565462a7
--- /dev/null
+++ b/ydb/library/yql/providers/yt/lib/yt_url_lister/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,22 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-lib-yt_url_lister)
+target_link_libraries(yt-lib-yt_url_lister PUBLIC
+  contrib-libs-cxxsupp
+  yutil
+  library-cpp-cgiparam
+  core-url_lister-interface
+  yt-lib-init_yt_api
+  yql-utils-log
+  cpp-mapreduce-interface
+)
+target_sources(yt-lib-yt_url_lister PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/lib/yt_url_lister/yt_url_lister.cpp
+)
diff --git a/ydb/library/yql/providers/yt/lib/yt_url_lister/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/yt/lib/yt_url_lister/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..881f7f1bff
--- /dev/null
+++ b/ydb/library/yql/providers/yt/lib/yt_url_lister/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,23 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-lib-yt_url_lister)
+target_link_libraries(yt-lib-yt_url_lister PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+  library-cpp-cgiparam
+  core-url_lister-interface
+  yt-lib-init_yt_api
+  yql-utils-log
+  cpp-mapreduce-interface
+)
+target_sources(yt-lib-yt_url_lister PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/lib/yt_url_lister/yt_url_lister.cpp
+)
diff --git a/ydb/library/yql/providers/yt/lib/yt_url_lister/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/yt/lib/yt_url_lister/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..881f7f1bff
--- /dev/null
+++ b/ydb/library/yql/providers/yt/lib/yt_url_lister/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,23 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-lib-yt_url_lister)
+target_link_libraries(yt-lib-yt_url_lister PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+  library-cpp-cgiparam
+  core-url_lister-interface
+  yt-lib-init_yt_api
+  yql-utils-log
+  cpp-mapreduce-interface
+)
+target_sources(yt-lib-yt_url_lister PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/lib/yt_url_lister/yt_url_lister.cpp
+)
diff --git a/ydb/library/yql/providers/yt/lib/yt_url_lister/CMakeLists.txt b/ydb/library/yql/providers/yt/lib/yt_url_lister/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/ydb/library/yql/providers/yt/lib/yt_url_lister/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+  include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+  include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+  include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+  include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/providers/yt/lib/yt_url_lister/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/yt/lib/yt_url_lister/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..83565462a7
--- /dev/null
+++ b/ydb/library/yql/providers/yt/lib/yt_url_lister/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,22 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yt-lib-yt_url_lister)
+target_link_libraries(yt-lib-yt_url_lister PUBLIC
+  contrib-libs-cxxsupp
+  yutil
+  library-cpp-cgiparam
+  core-url_lister-interface
+  yt-lib-init_yt_api
+  yql-utils-log
+  cpp-mapreduce-interface
+)
+target_sources(yt-lib-yt_url_lister PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/yt/lib/yt_url_lister/yt_url_lister.cpp
+)
diff --git a/ydb/library/yql/providers/yt/lib/yt_url_lister/ya.make b/ydb/library/yql/providers/yt/lib/yt_url_lister/ya.make
new file mode 100644
index 0000000000..4600454b86
--- /dev/null
+++ b/ydb/library/yql/providers/yt/lib/yt_url_lister/ya.make
@@ -0,0 +1,15 @@
+LIBRARY()
+
+SRCS(
+    yt_url_lister.cpp
+)
+
+PEERDIR(
+    library/cpp/cgiparam
+    ydb/library/yql/core/url_lister/interface
+    ydb/library/yql/providers/yt/lib/init_yt_api
+    ydb/library/yql/utils/log
+    yt/cpp/mapreduce/interface
+)
+
+END()
diff --git a/ydb/library/yql/providers/yt/lib/yt_url_lister/yt_url_lister.cpp b/ydb/library/yql/providers/yt/lib/yt_url_lister/yt_url_lister.cpp
new file mode 100644
index 0000000000..b454907572
--- /dev/null
+++ b/ydb/library/yql/providers/yt/lib/yt_url_lister/yt_url_lister.cpp
@@ -0,0 +1,111 @@
+#include "yt_url_lister.h"
+
+#include <library/cpp/cgiparam/cgiparam.h>
+
+#include <ydb/library/yql/providers/yt/lib/init_yt_api/init.h>
+#include <ydb/library/yql/utils/log/log.h>
+
+#include <yt/cpp/mapreduce/interface/client.h>
+
+#include <util/generic/guid.h>
+#include <util/generic/strbuf.h>
+#include <util/string/builder.h>
+
+
+namespace NYql::NPrivate {
+
+const char Sep = '/';
+const TStringBuf Scheme = "yt";
+
+class TYtUrlLister: public IUrlLister {
+public:
+    TYtUrlLister() = default;
+
+public:
+    bool Accept(const THttpURL& url) const override {
+        auto rawScheme = url.GetField(NUri::TField::FieldScheme);
+        return NUri::EqualNoCase(rawScheme, Scheme);
+    }
+
+    TVector<TUrlListEntry> ListUrl(const THttpURL& url, const TString& token) const override {
+        InitYtApiOnce();
+
+        TCgiParameters params(url.GetField(NUri::TField::FieldQuery));
+
+        NYT::TCreateClientOptions createOpts;
+        if (token) {
+            createOpts.Token(token);
+        }
+
+        auto host = url.PrintS(NUri::TField::FlagHostPort);
+
+        auto path = params.Has("path")
+            ? params.Get("path")
+            : TString(TStringBuf(url.GetField(NUri::TField::FieldPath)).Skip(1));
+
+        auto client = NYT::CreateClient(host, createOpts);
+        NYT::IClientBasePtr tx = client;
+
+        TString txId = params.Get("transaction_id");
+        if (!txId) {
+            txId = params.Get("t");
+        }
+
+        YQL_LOG(INFO) << "YtUrlLister: host=" << host << ", path='" << path << "', tx=" << txId;
+
+        if (txId) {
+            TGUID guid;
+            if (!GetGuid(txId, guid)) {
+                ythrow yexception() << "Bad transaction ID: " << txId;
+            }
+
+            tx = client->AttachTransaction(guid);
+        }
+
+        auto composeUrl = [&](auto name) {
+            THttpURL url;
+
+            url.Set(NUri::TField::FieldScheme, Scheme);
+            url.Set(NUri::TField::FieldHost, host);
+            url.Set(NUri::TField::FieldPath, TStringBuilder() << Sep << path << Sep << name);
+
+            if (txId) {
+                url.Set(NUri::TField::FieldQuery, TStringBuilder() << "transaction_id=" << txId);
+            }
+
+            return url;
+        };
+
+        NYT::TListOptions listOpts;
+        listOpts.AttributeFilter(
+            NYT::TAttributeFilter().Attributes({"type"})
+        );
+
+        TVector<TUrlListEntry> entries;
+
+        for (const auto& item: tx->List(path, listOpts)) {
+            auto& entry = entries.emplace_back();
+
+            const auto& itemName = item.AsString();
+            const auto& itemType = item.GetAttributes()["type"].AsString();
+
+            entry.Name = itemName;
+            entry.Url = composeUrl(itemName);
+            entry.Type = itemType == "map_node"
+                ? EUrlListEntryType::DIRECTORY
+                : EUrlListEntryType::FILE;
+        }
+
+        return entries;
+    }
+};
+
+}
+
+namespace NYql {
+
+IUrlListerPtr MakeYtUrlLister() {
+    return MakeIntrusive<NPrivate::TYtUrlLister>();
+}
+
+}
diff --git a/ydb/library/yql/providers/yt/lib/yt_url_lister/yt_url_lister.h b/ydb/library/yql/providers/yt/lib/yt_url_lister/yt_url_lister.h
new file mode 100644
index 0000000000..a2636691f8
--- /dev/null
+++ b/ydb/library/yql/providers/yt/lib/yt_url_lister/yt_url_lister.h
@@ -0,0 +1,10 @@
+#pragma once
+
+#include <ydb/library/yql/core/url_lister/interface/url_lister.h>
+
+
+namespace NYql {
+
+IUrlListerPtr MakeYtUrlLister();
+
+}
diff --git a/ydb/library/yql/providers/yt/provider/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/yt/provider/CMakeLists.darwin-x86_64.txt
index 6c40d166a9..011d8774a5 100644
--- a/ydb/library/yql/providers/yt/provider/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/providers/yt/provider/CMakeLists.darwin-x86_64.txt
@@ -38,6 +38,7 @@ target_link_libraries(providers-yt-provider PUBLIC
   yql-core-peephole_opt
   yql-core-type_ann
   yql-core-file_storage
+  core-url_lister-interface
   yql-dq-integration
   yql-dq-opt
   library-yql-minikql
diff --git a/ydb/library/yql/providers/yt/provider/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/yt/provider/CMakeLists.linux-aarch64.txt
index b4dbad2e59..5fed4af2d0 100644
--- a/ydb/library/yql/providers/yt/provider/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/providers/yt/provider/CMakeLists.linux-aarch64.txt
@@ -39,6 +39,7 @@ target_link_libraries(providers-yt-provider PUBLIC
   yql-core-peephole_opt
   yql-core-type_ann
   yql-core-file_storage
+  core-url_lister-interface
   yql-dq-integration
   yql-dq-opt
   library-yql-minikql
diff --git a/ydb/library/yql/providers/yt/provider/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/yt/provider/CMakeLists.linux-x86_64.txt
index b4dbad2e59..5fed4af2d0 100644
--- a/ydb/library/yql/providers/yt/provider/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/providers/yt/provider/CMakeLists.linux-x86_64.txt
@@ -39,6 +39,7 @@ target_link_libraries(providers-yt-provider PUBLIC
   yql-core-peephole_opt
   yql-core-type_ann
   yql-core-file_storage
+  core-url_lister-interface
   yql-dq-integration
   yql-dq-opt
   library-yql-minikql
diff --git a/ydb/library/yql/providers/yt/provider/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/yt/provider/CMakeLists.windows-x86_64.txt
index 6c40d166a9..011d8774a5 100644
--- a/ydb/library/yql/providers/yt/provider/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/providers/yt/provider/CMakeLists.windows-x86_64.txt
@@ -38,6 +38,7 @@ target_link_libraries(providers-yt-provider PUBLIC
   yql-core-peephole_opt
   yql-core-type_ann
   yql-core-file_storage
+  core-url_lister-interface
   yql-dq-integration
   yql-dq-opt
   library-yql-minikql
diff --git a/ydb/library/yql/providers/yt/provider/ut/yql_yt_dq_integration_ut.cpp b/ydb/library/yql/providers/yt/provider/ut/yql_yt_dq_integration_ut.cpp
index fd02e155f0..7b74a8b7e6 100644
--- a/ydb/library/yql/providers/yt/provider/ut/yql_yt_dq_integration_ut.cpp
+++ b/ydb/library/yql/providers/yt/provider/ut/yql_yt_dq_integration_ut.cpp
@@ -32,7 +32,7 @@ Y_UNIT_TEST_SUITE(TSchedulerTest) {
         UNIT_ASSERT(astRes.IsOk());
         TExprContext exprCtx_;
         TExprNode::TPtr exprRoot_;
-        UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot_, exprCtx_, nullptr));
+        UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot_, exprCtx_, nullptr, nullptr));
         TString cluster;
         const auto result = dqIntegration->Partition(settings, maxTasks, *exprRoot_, partitions, &cluster, exprCtx_, false);
         const auto expected = 428;
@@ -73,7 +73,7 @@ Y_UNIT_TEST_SUITE(TSchedulerTest) {
         UNIT_ASSERT(astRes.IsOk());
         TExprContext exprCtx_;
         TExprNode::TPtr exprRoot_;
-        UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot_, exprCtx_, nullptr));
+        UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot_, exprCtx_, nullptr, nullptr));
         TString cluster;
         const auto result = dqIntegration->Partition(settings, maxTasks, *exprRoot_, partitions, &cluster, exprCtx_, false);
         const auto expected = 642;
@@ -114,7 +114,7 @@ Y_UNIT_TEST_SUITE(TSchedulerTest) {
         UNIT_ASSERT(astRes.IsOk());
         TExprContext exprCtx_;
         TExprNode::TPtr exprRoot_;
-        UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot_, exprCtx_, nullptr));
+        UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot_, exprCtx_, nullptr, nullptr));
         TString cluster;
         const auto result = dqIntegration->Partition(settings, maxTasks, *exprRoot_, partitions, &cluster, exprCtx_, false);
         const auto expected = 214;
diff --git a/ydb/library/yql/providers/yt/provider/ut/yql_yt_epoch_ut.cpp b/ydb/library/yql/providers/yt/provider/ut/yql_yt_epoch_ut.cpp
index 639f6f04ad..ed0ed460b1 100644
--- a/ydb/library/yql/providers/yt/provider/ut/yql_yt_epoch_ut.cpp
+++ b/ydb/library/yql/providers/yt/provider/ut/yql_yt_epoch_ut.cpp
@@ -36,7 +36,7 @@ Y_UNIT_TEST_SUITE(TYqlEpoch) {
         UNIT_ASSERT(astRes.IsOk());
         TExprContext exprCtx;
         TExprNode::TPtr exprRoot;
-        UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr));
+        UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr, nullptr));
 
         auto functionRegistry = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry());
         TTestTablesMapping testTables;
diff --git a/ydb/library/yql/providers/yt/provider/ya.make b/ydb/library/yql/providers/yt/provider/ya.make
index a6973aca2a..c0a858a1b3 100644
--- a/ydb/library/yql/providers/yt/provider/ya.make
+++ b/ydb/library/yql/providers/yt/provider/ya.make
@@ -60,6 +60,7 @@ PEERDIR(
     ydb/library/yql/core/peephole_opt
     ydb/library/yql/core/type_ann
     ydb/library/yql/core/file_storage
+    ydb/library/yql/core/url_lister/interface
     ydb/library/yql/dq/integration
     ydb/library/yql/dq/opt
     ydb/library/yql/minikql
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_load_table_meta.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_load_table_meta.cpp
index 05676e48e5..496efc48a3 100644
--- a/ydb/library/yql/providers/yt/provider/yql_yt_load_table_meta.cpp
+++ b/ydb/library/yql/providers/yt/provider/yql_yt_load_table_meta.cpp
@@ -100,7 +100,10 @@ public:
                     // skip if table already has loaded metadata and has only read intents
                     if (State_->Types->IsReadOnly || State_->Types->UseTableMetaFromGraph || tableDesc.HasWriteLock || !HasModifyIntents(tableDesc.Intents)) {
                         // Intents/views can be updated since evaluation phase
-                        if (!tableDesc.FillViews(clusterAndTable.first, clusterAndTable.second, ctx, State_->Types->Modules.get(), *State_->Types->RandomProvider)) {
+                        if (!tableDesc.FillViews(
+                            clusterAndTable.first, clusterAndTable.second, ctx,
+                            State_->Types->Modules.get(), State_->Types->UrlListerManager.Get(), *State_->Types->RandomProvider
+                        )) {
                             return TStatus::Error;
                         }
                         continue;
@@ -224,7 +227,10 @@ public:
                 }
 
                 if (0 == LoadCtx->Epoch) {
-                    if (!tableDesc.Fill(cluster, tableName, ctx, State_->Types->Modules.get(), *State_->Types->RandomProvider)) {
+                    if (!tableDesc.Fill(
+                        cluster, tableName, ctx,
+                        State_->Types->Modules.get(), State_->Types->UrlListerManager.Get(), *State_->Types->RandomProvider
+                    )) {
                         return TStatus::Error;
                     }
                 }
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_provider.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_provider.cpp
index ec89019eb5..81116065b5 100644
--- a/ydb/library/yql/providers/yt/provider/yql_yt_provider.cpp
+++ b/ydb/library/yql/providers/yt/provider/yql_yt_provider.cpp
@@ -12,7 +12,9 @@
 
 namespace NYql {
 
-bool TYtTableDescription::Fill(const TString& cluster, const TString& table, TExprContext& ctx, IModuleResolver* moduleResolver, IRandomProvider& randomProvider) {
+bool TYtTableDescription::Fill(
+    const TString& cluster, const TString& table, TExprContext& ctx,
+    IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider) {
     const TStructExprType* type = RowSpec ? RowSpec->GetType() : nullptr;
     if (!type) {
         TVector<const TItemExprType*> items;
@@ -25,7 +27,8 @@ bool TYtTableDescription::Fill(const TString& cluster, const TString& table, TEx
     }
 
     if (!TYtTableDescriptionBase::Fill(TString{YtProviderName}, cluster,
-        table, type, Meta->SqlView, Meta->SqlViewSyntaxVersion, Meta->Attrs, ctx, moduleResolver, randomProvider)) {
+        table, type, Meta->SqlView, Meta->SqlViewSyntaxVersion, Meta->Attrs, ctx,
+        moduleResolver, urlListerManager, randomProvider)) {
         return false;
     }
     if (QB2RowSpec) {
@@ -236,8 +239,12 @@ void TYtTableDescription::SetConstraintsReady() {
     }
 }
 
-bool TYtTableDescription::FillViews(const TString& cluster, const TString& table, TExprContext& ctx, IModuleResolver* moduleResolver, IRandomProvider& randomProvider) {
-    return TYtTableDescriptionBase::FillViews(TString{YtProviderName}, cluster, table, Meta->Attrs, ctx, moduleResolver, randomProvider);
+bool TYtTableDescription::FillViews(
+    const TString& cluster, const TString& table, TExprContext& ctx,
+    IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider) {
+    return TYtTableDescriptionBase::FillViews(
+        TString{YtProviderName}, cluster, table, Meta->Attrs, ctx,
+        moduleResolver, urlListerManager, randomProvider);
 }
 
 const TYtTableDescription& TYtTablesData::GetTable(const TString& cluster, const TString& table, TMaybe<ui32> epoch) const {
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_provider.h b/ydb/library/yql/providers/yt/provider/yql_yt_provider.h
index ead2fab597..24d76a4fe7 100644
--- a/ydb/library/yql/providers/yt/provider/yql_yt_provider.h
+++ b/ydb/library/yql/providers/yt/provider/yql_yt_provider.h
@@ -43,12 +43,16 @@ struct TYtTableDescription: public TYtTableDescriptionBase {
     size_t WriteValidateCount = 0;
     TMaybe<TString> Hash;
 
-    bool Fill(const TString& cluster, const TString& table, TExprContext& ctx, IModuleResolver* moduleResolver, IRandomProvider& randomProvider);
+    bool Fill(
+        const TString& cluster, const TString& table, TExprContext& ctx,
+        IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider);
     void ToYson(NYson::TYsonWriter& writer, const TString& cluster, const TString& table, const TString& view) const;
     bool Validate(TPosition pos, TStringBuf cluster, TStringBuf tableName, bool withQB,
         const THashMap<std::pair<TString, TString>, TString>& anonymousLabels, TExprContext& ctx) const;
     void SetConstraintsReady();
-    bool FillViews(const TString& cluster, const TString& table, TExprContext& ctx, IModuleResolver* moduleResolver, IRandomProvider& randomProvider);
+    bool FillViews(
+        const TString& cluster, const TString& table, TExprContext& ctx,
+        IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider);
 };
 
 // Anonymous tables are kept by labels
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_table_desc.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_table_desc.cpp
index 0aad683f34..609820e173 100644
--- a/ydb/library/yql/providers/yt/provider/yql_yt_table_desc.cpp
+++ b/ydb/library/yql/providers/yt/provider/yql_yt_table_desc.cpp
@@ -148,7 +148,7 @@ TExprNode::TPtr BuildIgnoreTypeV3Remapper(const TStructExprType* rowType, TExprC
 }
 
 TExprNode::TPtr CompileViewSql(const TString& provider, const TString& cluster, const TString& sql, ui16 syntaxVersion,
-    TExprContext& ctx, IModuleResolver* moduleResolver, IRandomProvider& randomProvider)
+    TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider)
 {
     NSQLTranslation::TTranslationSettings settings;
     settings.Mode = NSQLTranslation::ESqlMode::LIMITED_VIEW;
@@ -164,7 +164,7 @@ TExprNode::TPtr CompileViewSql(const TString& provider, const TString& cluster,
     }
 
     TExprNode::TPtr exprRoot;
-    if (!CompileExpr(*sqlRes.Root, exprRoot, ctx, moduleResolver, false, Max<ui32>(), syntaxVersion)) {
+    if (!CompileExpr(*sqlRes.Root, exprRoot, ctx, moduleResolver, urlListerManager, false, Max<ui32>(), syntaxVersion)) {
         return {};
     }
 
@@ -233,10 +233,10 @@ TExprNode::TPtr CompileViewSql(const TString& provider, const TString& cluster,
 
 
 bool TYtViewDescription::Fill(const TString& provider, const TString& cluster, const TString& sql, ui16 syntaxVersion, TExprContext& ctx,
-    IModuleResolver* moduleResolver, IRandomProvider& randomProvider)
+    IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider)
 {
     Sql = sql;
-    CompiledSql = CompileViewSql(provider, cluster, sql, syntaxVersion, ctx, moduleResolver, randomProvider);
+    CompiledSql = CompileViewSql(provider, cluster, sql, syntaxVersion, ctx, moduleResolver, urlListerManager, randomProvider);
     return bool(CompiledSql);
 }
 
@@ -247,7 +247,7 @@ void TYtViewDescription::CleanupCompiledSQL()
 
 bool TYtTableDescriptionBase::Fill(const TString& provider, const TString& cluster, const TString& table,
     const TStructExprType* type, const TString& viewSql, ui16 syntaxVersion, const THashMap<TString, TString>& metaAttrs,
-    TExprContext& ctx, IModuleResolver* moduleResolver, IRandomProvider& randomProvider)
+    TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider)
 {
     // (1) row type
     RawRowType = type;
@@ -326,13 +326,13 @@ bool TYtTableDescriptionBase::Fill(const TString& provider, const TString& clust
     }
 
     // (3) views
-    if (!FillViews(provider, cluster, table, metaAttrs, ctx, moduleResolver, randomProvider)) {
+    if (!FillViews(provider, cluster, table, metaAttrs, ctx, moduleResolver, urlListerManager, randomProvider)) {
         return false;
     }
 
     if (viewSql) {
         if (!View) {
-            if (!View.ConstructInPlace().Fill(provider, cluster, viewSql, syntaxVersion, ctx, moduleResolver, randomProvider)) {
+            if (!View.ConstructInPlace().Fill(provider, cluster, viewSql, syntaxVersion, ctx, moduleResolver, urlListerManager, randomProvider)) {
                 ctx.AddError(TIssue(TPosition(),
                     TStringBuilder() << "Can't load sql view, table: " << cluster << '.' << table));
                 return false;
@@ -344,7 +344,8 @@ bool TYtTableDescriptionBase::Fill(const TString& provider, const TString& clust
 }
 
 bool TYtTableDescriptionBase::FillViews(const TString& provider, const TString& cluster, const TString& table,
-    const THashMap<TString, TString>& metaAttrs, TExprContext& ctx, IModuleResolver* moduleResolver, IRandomProvider& randomProvider)
+    const THashMap<TString, TString>& metaAttrs, TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager,
+    IRandomProvider& randomProvider)
 {
     for (auto& view: Views) {
         TYtViewDescription& viewDesc = view.second;
@@ -372,7 +373,7 @@ bool TYtTableDescriptionBase::FillViews(const TString& provider, const TString&
                 }
             }
 
-            if (!viewDesc.Fill(provider, cluster, viewSql, syntaxVersion, ctx, moduleResolver, randomProvider)) {
+            if (!viewDesc.Fill(provider, cluster, viewSql, syntaxVersion, ctx, moduleResolver, urlListerManager, randomProvider)) {
                 ctx.AddError(TIssue(TPosition(),
                     TStringBuilder() << "Can't load sql view " << viewSql.Quote()
                     << ", table: " << cluster << '.' << table
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_table_desc.h b/ydb/library/yql/providers/yt/provider/yql_yt_table_desc.h
index 58432d3e97..3c47e70fbd 100644
--- a/ydb/library/yql/providers/yt/provider/yql_yt_table_desc.h
+++ b/ydb/library/yql/providers/yt/provider/yql_yt_table_desc.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <ydb/library/yql/ast/yql_expr.h>
+#include <ydb/library/yql/core/url_lister/interface/url_lister_manager.h>
 
 #include <library/cpp/random_provider/random_provider.h>
 
@@ -48,7 +49,7 @@ struct TYtViewDescription {
     const TTypeAnnotationNode* RowType = nullptr; // Filled only if scheme requested
 
     bool Fill(const TString& provider, const TString& cluster, const TString& sql, ui16 syntaxVersion, TExprContext& ctx,
-        IModuleResolver* moduleResolver, IRandomProvider& randomProvider);
+        IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider);
     void CleanupCompiledSQL();
 };
 
@@ -69,10 +70,10 @@ struct TYtTableDescriptionBase {
 
     bool Fill(const TString& provider, const TString& cluster, const TString& table, const TStructExprType* type,
         const TString& viewSql, ui16 syntaxVersion, const THashMap<TString, TString>& metaAttrs, TExprContext& ctx,
-        IModuleResolver* moduleResolver, IRandomProvider& randomProvider);
+        IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider);
     void CleanupCompiledSQL();
     bool FillViews(const TString& provider, const TString& cluster, const TString& table, const THashMap<TString, TString>& metaAttrs,
-        TExprContext& ctx, IModuleResolver* moduleResolver, IRandomProvider& randomProvider);
+        TExprContext& ctx, IModuleResolver* moduleResolver, IUrlListerManager* urlListerManager, IRandomProvider& randomProvider);
 };
 
 }
diff --git a/ydb/library/yql/public/embedded/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/public/embedded/CMakeLists.darwin-x86_64.txt
index a50913ea6b..25dad8fc57 100644
--- a/ydb/library/yql/public/embedded/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/public/embedded/CMakeLists.darwin-x86_64.txt
@@ -38,9 +38,11 @@ target_link_libraries(yql-public-embedded PUBLIC
   providers-common-proto
   providers-common-udf_resolve
   yql-core-url_preprocessing
+  yql-core-url_lister
   yt-gateway-native
   yt-lib-log
   yt-lib-yt_download
+  yt-lib-yt_url_lister
   providers-yt-provider
 )
 target_sources(yql-public-embedded PRIVATE
diff --git a/ydb/library/yql/public/embedded/CMakeLists.linux-aarch64.txt b/ydb/library/yql/public/embedded/CMakeLists.linux-aarch64.txt
index fb20043f75..747f38d1a5 100644
--- a/ydb/library/yql/public/embedded/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/public/embedded/CMakeLists.linux-aarch64.txt
@@ -39,9 +39,11 @@ target_link_libraries(yql-public-embedded PUBLIC
   providers-common-proto
   providers-common-udf_resolve
   yql-core-url_preprocessing
+  yql-core-url_lister
   yt-gateway-native
   yt-lib-log
   yt-lib-yt_download
+  yt-lib-yt_url_lister
   providers-yt-provider
 )
 target_sources(yql-public-embedded PRIVATE
diff --git a/ydb/library/yql/public/embedded/CMakeLists.linux-x86_64.txt b/ydb/library/yql/public/embedded/CMakeLists.linux-x86_64.txt
index fb20043f75..747f38d1a5 100644
--- a/ydb/library/yql/public/embedded/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/public/embedded/CMakeLists.linux-x86_64.txt
@@ -39,9 +39,11 @@ target_link_libraries(yql-public-embedded PUBLIC
   providers-common-proto
   providers-common-udf_resolve
   yql-core-url_preprocessing
+  yql-core-url_lister
   yt-gateway-native
   yt-lib-log
   yt-lib-yt_download
+  yt-lib-yt_url_lister
   providers-yt-provider
 )
 target_sources(yql-public-embedded PRIVATE
diff --git a/ydb/library/yql/public/embedded/CMakeLists.windows-x86_64.txt b/ydb/library/yql/public/embedded/CMakeLists.windows-x86_64.txt
index a50913ea6b..25dad8fc57 100644
--- a/ydb/library/yql/public/embedded/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/public/embedded/CMakeLists.windows-x86_64.txt
@@ -38,9 +38,11 @@ target_link_libraries(yql-public-embedded PUBLIC
   providers-common-proto
   providers-common-udf_resolve
   yql-core-url_preprocessing
+  yql-core-url_lister
   yt-gateway-native
   yt-lib-log
   yt-lib-yt_download
+  yt-lib-yt_url_lister
   providers-yt-provider
 )
 target_sources(yql-public-embedded PRIVATE
diff --git a/ydb/library/yql/public/embedded/ya.make b/ydb/library/yql/public/embedded/ya.make
index b740e1f1dc..d5c5f91575 100644
--- a/ydb/library/yql/public/embedded/ya.make
+++ b/ydb/library/yql/public/embedded/ya.make
@@ -30,9 +30,11 @@ PEERDIR(
     ydb/library/yql/providers/common/proto
     ydb/library/yql/providers/common/udf_resolve
     ydb/library/yql/core/url_preprocessing
+    ydb/library/yql/core/url_lister
     ydb/library/yql/providers/yt/gateway/native
     ydb/library/yql/providers/yt/lib/log
     ydb/library/yql/providers/yt/lib/yt_download
+    ydb/library/yql/providers/yt/lib/yt_url_lister
     ydb/library/yql/providers/yt/provider
 )
 
diff --git a/ydb/library/yql/public/embedded/yql_embedded.cpp b/ydb/library/yql/public/embedded/yql_embedded.cpp
index 102f302269..df2b2dde21 100644
--- a/ydb/library/yql/public/embedded/yql_embedded.cpp
+++ b/ydb/library/yql/public/embedded/yql_embedded.cpp
@@ -2,6 +2,7 @@
 
 #include <ydb/library/yql/providers/yt/lib/log/yt_logger.h>
 #include <ydb/library/yql/providers/yt/lib/yt_download/yt_download.h>
+#include <ydb/library/yql/providers/yt/lib/yt_url_lister/yt_url_lister.h>
 #include <ydb/library/yql/providers/yt/gateway/native/yql_yt_native.h>
 #include <ydb/library/yql/providers/yt/provider/yql_yt_provider.h>
 
@@ -19,6 +20,7 @@
 #include <ydb/library/yql/core/file_storage/file_storage.h>
 #include <ydb/library/yql/core/file_storage/proto/file_storage.pb.h>
 #include <ydb/library/yql/core/file_storage/http_download/http_download.h>
+#include <ydb/library/yql/core/url_lister/url_lister_manager.h>
 #include <ydb/library/yql/core/services/mounts/yql_mounts.h>
 #include <ydb/library/yql/utils/log/log.h>
 #include <ydb/library/yql/utils/backtrace/backtrace.h>
@@ -356,6 +358,11 @@ namespace NYql {
                 ProgramFactory_->SetGatewaysConfig(&GatewaysConfig_);
                 ProgramFactory_->SetFileStorage(FileStorage_);
                 ProgramFactory_->SetUrlPreprocessing(MakeIntrusive<TUrlPreprocessing>(GatewaysConfig_));
+                ProgramFactory_->SetUrlListerManager(
+                    MakeUrlListerManager(
+                        {MakeYtUrlLister()}
+                    )
+                );
             }
 
             THolder<IOperation> Run(const TString& queryText, const TOperationOptions& options) const override {
diff --git a/ydb/library/yql/public/fastcheck/fastcheck.cpp b/ydb/library/yql/public/fastcheck/fastcheck.cpp
index ec54aef5da..4e7043c185 100644
--- a/ydb/library/yql/public/fastcheck/fastcheck.cpp
+++ b/ydb/library/yql/public/fastcheck/fastcheck.cpp
@@ -82,7 +82,7 @@ bool CheckProgram(const TString& program, const TOptions& options, TIssues& erro
 
     TExprContext exprCtx(libCtx.NextUniqueId);
     TExprNode::TPtr exprRoot;
-    if (!CompileExpr(*astRes.Root, exprRoot, exprCtx, moduleResolver.get(), false, Max<ui32>(), options.SyntaxVersion)) {
+    if (!CompileExpr(*astRes.Root, exprRoot, exprCtx, moduleResolver.get(), nullptr, false, Max<ui32>(), options.SyntaxVersion)) {
         errors = exprCtx.IssueManager.GetIssues();
         exprCtx.IssueManager.Reset();
         return false;
diff --git a/ydb/library/yql/public/purecalc/common/worker_factory.cpp b/ydb/library/yql/public/purecalc/common/worker_factory.cpp
index 223dee8c1b..6a5ddbe37c 100644
--- a/ydb/library/yql/public/purecalc/common/worker_factory.cpp
+++ b/ydb/library/yql/public/purecalc/common/worker_factory.cpp
@@ -177,7 +177,7 @@ TExprNode::TPtr TWorkerFactory<TBase>::Compile(
     // Translate AST into expression
 
     TExprNode::TPtr exprRoot;
-    if (!CompileExpr(*astRes.Root, exprRoot, ExprContext_, moduleResolver.get(), 0, syntaxVersion)) {
+    if (!CompileExpr(*astRes.Root, exprRoot, ExprContext_, moduleResolver.get(), nullptr, 0, syntaxVersion)) {
         TStringStream astStr;
         astRes.Root->PrettyPrintTo(astStr, TAstPrintFlags::ShortQuote | TAstPrintFlags::PerLine);
         ythrow TCompileError(astStr.Str(), ExprContext_.IssueManager.GetIssues().ToString()) << "failed to compile";
diff --git a/ydb/library/yql/sql/v1/context.h b/ydb/library/yql/sql/v1/context.h
index f69449d9fa..9d649d80c9 100644
--- a/ydb/library/yql/sql/v1/context.h
+++ b/ydb/library/yql/sql/v1/context.h
@@ -281,6 +281,16 @@ namespace NSQLTranslationV1 {
         using TLiteralWithPosition = std::pair<TString, TPosition>;
         using TLibraryStuff = std::tuple<TPosition, std::optional<TLiteralWithPosition>, std::optional<TLiteralWithPosition>>;
         std::unordered_map<TString, TLibraryStuff> Libraries; // alias -> optional file with token
+        using TPackageStuff = std::tuple<
+            TPosition, TLiteralWithPosition,
+            std::optional<TLiteralWithPosition>
+        >;
+
+        std::unordered_map<TString, TPackageStuff> Packages; // alias -> url with optional token
+
+        using TOverrideLibraryStuff = std::tuple<TPosition>;
+        std::unordered_map<TString, TOverrideLibraryStuff> OverrideLibraries; // alias -> position
+
         THashMap<TString, ui32> PackageVersions;
         NYql::TWarningPolicy WarningPolicy;
         TString PqReadByRtmrCluster;
diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp
index cb93d8bfe4..de0ec128ee 100644
--- a/ydb/library/yql/sql/v1/query.cpp
+++ b/ydb/library/yql/sql/v1/query.cpp
@@ -2345,6 +2345,37 @@ public:
                 Add(Y("declare", var.first, var.second));
             }
 
+            for (const auto& overrideLibrary: ctx.OverrideLibraries) {
+                auto node = Y(
+                    "override_library",
+                    new TAstAtomNodeImpl(
+                        std::get<TPosition>(overrideLibrary.second),
+                        overrideLibrary.first, TNodeFlags::ArbitraryContent
+                    ));
+
+                Add(node);
+            }
+
+            for (const auto& package: ctx.Packages) {
+                const auto& [url, urlPosition] = std::get<1U>(package.second);
+
+                auto node = Y(
+                    "package",
+                    new TAstAtomNodeImpl(
+                        std::get<TPosition>(package.second), package.first,
+                        TNodeFlags::ArbitraryContent
+                    ),
+                    new TAstAtomNodeImpl(urlPosition, url, TNodeFlags::ArbitraryContent));
+
+                if (const auto& tokenWithPosition = std::get<2U>(package.second)) {
+                    const auto& [token, tokenPosition] = *tokenWithPosition;
+
+                    node = L(node, new TAstAtomNodeImpl(tokenPosition, token, TNodeFlags::ArbitraryContent));
+                }
+
+                Add(node);
+            }
+
             for (const auto& lib : ctx.Libraries) {
                 auto node = Y("library", new TAstAtomNodeImpl(std::get<TPosition>(lib.second), lib.first, TNodeFlags::ArbitraryContent));
                 if (const auto& first = std::get<1U>(lib.second)) {
diff --git a/ydb/library/yql/sql/v1/sql_query.cpp b/ydb/library/yql/sql/v1/sql_query.cpp
index 77e881b938..0adae5cb6e 100644
--- a/ydb/library/yql/sql/v1/sql_query.cpp
+++ b/ydb/library/yql/sql/v1/sql_query.cpp
@@ -1526,6 +1526,91 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
 
             Ctx.Libraries[alias] = std::move(library);
             Ctx.IncrementMonCounter("sql_pragma", "library");
+        } else if (normalizedPragma == "package") {
+            if (values.size() < 2U || values.size() > 3U) {
+                Error() << "Expected package name, url and optional token name as pragma values";
+                Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
+                return {};
+            }
+
+            TString packageName;
+            if (!values.front().GetLiteral(packageName, Ctx)) {
+                Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
+                return {};
+            }
+
+            TContext::TPackageStuff package;
+            std::get<TPosition>(package) = values.front().Build()->GetPos();
+
+            auto fillLiteral = [&](auto& literal, size_t index) {
+                if (values.size() <= index) {
+                    return true;
+                }
+
+                constexpr bool optional = std::is_base_of_v<
+                    std::optional<TContext::TLiteralWithPosition>,
+                    std::decay_t<decltype(literal)>
+                >;
+
+                TContext::TLiteralWithPosition* literalPtr;
+
+                if constexpr (optional) {
+                    literal.emplace();
+                    literalPtr = &*literal;
+                } else {
+                    literalPtr = &literal;
+                }
+
+                literalPtr->second = values[index].Build()->GetPos();
+
+                if (!values[index].GetLiteral(literalPtr->first, Ctx)) {
+                    Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
+                    return false;
+                }
+
+                return true;
+            };
+
+            // fill url
+            auto& urlLiteral = std::get<1U>(package);
+            if (!fillLiteral(urlLiteral, 1U)) {
+                return {};
+            }
+
+            TSet<TString> names;
+            SubstParameters(urlLiteral.first, Nothing(), &names);
+            for (const auto& name : names) {
+                auto namedNode = GetNamedNode(name);
+                if (!namedNode) {
+                    return {};
+                }
+            }
+
+            // fill token
+            if (!fillLiteral(std::get<2U>(package), 2U)) {
+                return {};
+            }
+
+            Ctx.Packages[packageName] = std::move(package);
+            Ctx.IncrementMonCounter("sql_pragma", "package");
+        } else if (normalizedPragma == "overridelibrary") {
+            if (values.size() != 1U) {
+                Error() << "Expected override library alias as pragma value";
+                Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
+                return {};
+            }
+
+            TString alias;
+            if (!values.front().GetLiteral(alias, Ctx)) {
+                Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
+                return {};
+            }
+
+            TContext::TOverrideLibraryStuff overrideLibrary;
+            std::get<TPosition>(overrideLibrary) = values.front().Build()->GetPos();
+
+            Ctx.OverrideLibraries[alias] = std::move(overrideLibrary);
+            Ctx.IncrementMonCounter("sql_pragma", "overridelibrary");
         } else if (normalizedPragma == "directread") {
             Ctx.PragmaDirectRead = true;
             Ctx.IncrementMonCounter("sql_pragma", "DirectRead");