aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorvvvv <vvvv@ydb.tech>2023-11-12 14:17:41 +0300
committervvvv <vvvv@ydb.tech>2023-11-12 14:32:51 +0300
commite4bf7da8cc01e5fd68c54a0d299c2d07c67b0076 (patch)
tree5ddd0290b679c00cbf6a450fe31db60c167349cc
parent33a303b290c6dec1b122a3c782f0b17fc6ea6413 (diff)
downloadydb-e4bf7da8cc01e5fd68c54a0d299c2d07c67b0076.tar.gz
YQL-16964 clone udf tools into ydb
-rw-r--r--.mapping.json15
-rw-r--r--ydb/library/yql/tools/CMakeLists.txt3
-rw-r--r--ydb/library/yql/tools/udf_dep_stub/CMakeLists.darwin-x86_64.txt25
-rw-r--r--ydb/library/yql/tools/udf_dep_stub/CMakeLists.linux-aarch64.txt30
-rw-r--r--ydb/library/yql/tools/udf_dep_stub/CMakeLists.linux-x86_64.txt30
-rw-r--r--ydb/library/yql/tools/udf_dep_stub/CMakeLists.txt17
-rw-r--r--ydb/library/yql/tools/udf_dep_stub/CMakeLists.windows-x86_64.txt18
-rw-r--r--ydb/library/yql/tools/udf_dep_stub/main.cpp0
-rw-r--r--ydb/library/yql/tools/udf_dep_stub/ya.make7
-rw-r--r--ydb/library/yql/tools/udf_probe/CMakeLists.darwin-x86_64.txt35
-rw-r--r--ydb/library/yql/tools/udf_probe/CMakeLists.linux-aarch64.txt38
-rw-r--r--ydb/library/yql/tools/udf_probe/CMakeLists.linux-x86_64.txt40
-rw-r--r--ydb/library/yql/tools/udf_probe/CMakeLists.txt17
-rw-r--r--ydb/library/yql/tools/udf_probe/CMakeLists.windows-x86_64.txt28
-rw-r--r--ydb/library/yql/tools/udf_probe/udf_probe.cpp35
-rw-r--r--ydb/library/yql/tools/udf_probe/ya.make21
-rw-r--r--ydb/library/yql/tools/udf_resolver/CMakeLists.darwin-x86_64.txt43
-rw-r--r--ydb/library/yql/tools/udf_resolver/CMakeLists.linux-aarch64.txt47
-rw-r--r--ydb/library/yql/tools/udf_resolver/CMakeLists.linux-x86_64.txt49
-rw-r--r--ydb/library/yql/tools/udf_resolver/CMakeLists.txt17
-rw-r--r--ydb/library/yql/tools/udf_resolver/CMakeLists.windows-x86_64.txt36
-rw-r--r--ydb/library/yql/tools/udf_resolver/discover.cpp144
-rw-r--r--ydb/library/yql/tools/udf_resolver/discover.h16
-rw-r--r--ydb/library/yql/tools/udf_resolver/udf_resolver.cpp459
-rw-r--r--ydb/library/yql/tools/udf_resolver/ya.make33
-rw-r--r--ydb/library/yql/tools/ya.make3
26 files changed, 1206 insertions, 0 deletions
diff --git a/.mapping.json b/.mapping.json
index b31a969bf0..5c586a131e 100644
--- a/.mapping.json
+++ b/.mapping.json
@@ -8475,6 +8475,21 @@
"ydb/library/yql/tools/sql_formatter/CMakeLists.linux-x86_64.txt":"",
"ydb/library/yql/tools/sql_formatter/CMakeLists.txt":"",
"ydb/library/yql/tools/sql_formatter/CMakeLists.windows-x86_64.txt":"",
+ "ydb/library/yql/tools/udf_dep_stub/CMakeLists.darwin-x86_64.txt":"",
+ "ydb/library/yql/tools/udf_dep_stub/CMakeLists.linux-aarch64.txt":"",
+ "ydb/library/yql/tools/udf_dep_stub/CMakeLists.linux-x86_64.txt":"",
+ "ydb/library/yql/tools/udf_dep_stub/CMakeLists.txt":"",
+ "ydb/library/yql/tools/udf_dep_stub/CMakeLists.windows-x86_64.txt":"",
+ "ydb/library/yql/tools/udf_probe/CMakeLists.darwin-x86_64.txt":"",
+ "ydb/library/yql/tools/udf_probe/CMakeLists.linux-aarch64.txt":"",
+ "ydb/library/yql/tools/udf_probe/CMakeLists.linux-x86_64.txt":"",
+ "ydb/library/yql/tools/udf_probe/CMakeLists.txt":"",
+ "ydb/library/yql/tools/udf_probe/CMakeLists.windows-x86_64.txt":"",
+ "ydb/library/yql/tools/udf_resolver/CMakeLists.darwin-x86_64.txt":"",
+ "ydb/library/yql/tools/udf_resolver/CMakeLists.linux-aarch64.txt":"",
+ "ydb/library/yql/tools/udf_resolver/CMakeLists.linux-x86_64.txt":"",
+ "ydb/library/yql/tools/udf_resolver/CMakeLists.txt":"",
+ "ydb/library/yql/tools/udf_resolver/CMakeLists.windows-x86_64.txt":"",
"ydb/library/yql/tools/yqlrun/CMakeLists.darwin-x86_64.txt":"",
"ydb/library/yql/tools/yqlrun/CMakeLists.linux-aarch64.txt":"",
"ydb/library/yql/tools/yqlrun/CMakeLists.linux-x86_64.txt":"",
diff --git a/ydb/library/yql/tools/CMakeLists.txt b/ydb/library/yql/tools/CMakeLists.txt
index 61ab79236b..1d7c5f669c 100644
--- a/ydb/library/yql/tools/CMakeLists.txt
+++ b/ydb/library/yql/tools/CMakeLists.txt
@@ -12,4 +12,7 @@ add_subdirectory(dqrun)
add_subdirectory(mrjob)
add_subdirectory(sql2yql)
add_subdirectory(sql_formatter)
+add_subdirectory(udf_dep_stub)
+add_subdirectory(udf_probe)
+add_subdirectory(udf_resolver)
add_subdirectory(yqlrun)
diff --git a/ydb/library/yql/tools/udf_dep_stub/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..6e0b1af6b7
--- /dev/null
+++ b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,25 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_shared_library(yql_udf_dep_stub)
+target_link_libraries(yql_udf_dep_stub PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+)
+target_link_options(yql_udf_dep_stub PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -undefined
+ dynamic_lookup
+ -fPIC
+)
+target_sources(yql_udf_dep_stub PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_dep_stub/main.cpp
+)
+vcs_info(yql_udf_dep_stub)
diff --git a/ydb/library/yql/tools/udf_dep_stub/CMakeLists.linux-aarch64.txt b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..4cc94a170c
--- /dev/null
+++ b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,30 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_shared_library(yql_udf_dep_stub)
+target_link_libraries(yql_udf_dep_stub PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+)
+target_link_options(yql_udf_dep_stub PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -Wl,-z,notext
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(yql_udf_dep_stub PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_dep_stub/main.cpp
+)
+vcs_info(yql_udf_dep_stub)
diff --git a/ydb/library/yql/tools/udf_dep_stub/CMakeLists.linux-x86_64.txt b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..4cc94a170c
--- /dev/null
+++ b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,30 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_shared_library(yql_udf_dep_stub)
+target_link_libraries(yql_udf_dep_stub PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+)
+target_link_options(yql_udf_dep_stub PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -Wl,-z,notext
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(yql_udf_dep_stub PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_dep_stub/main.cpp
+)
+vcs_info(yql_udf_dep_stub)
diff --git a/ydb/library/yql/tools/udf_dep_stub/CMakeLists.txt b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/tools/udf_dep_stub/CMakeLists.windows-x86_64.txt b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..17efeeeaf5
--- /dev/null
+++ b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,18 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_shared_library(yql_udf_dep_stub)
+target_link_libraries(yql_udf_dep_stub PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+)
+target_sources(yql_udf_dep_stub PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_dep_stub/main.cpp
+)
+vcs_info(yql_udf_dep_stub)
diff --git a/ydb/library/yql/tools/udf_dep_stub/main.cpp b/ydb/library/yql/tools/udf_dep_stub/main.cpp
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/ydb/library/yql/tools/udf_dep_stub/main.cpp
diff --git a/ydb/library/yql/tools/udf_dep_stub/ya.make b/ydb/library/yql/tools/udf_dep_stub/ya.make
new file mode 100644
index 0000000000..abe778ed12
--- /dev/null
+++ b/ydb/library/yql/tools/udf_dep_stub/ya.make
@@ -0,0 +1,7 @@
+DLL(yql_udf_dep_stub)
+
+SRCS(
+ main.cpp
+)
+
+END()
diff --git a/ydb/library/yql/tools/udf_probe/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/tools/udf_probe/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..1edbd44aeb
--- /dev/null
+++ b/ydb/library/yql/tools/udf_probe/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,35 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(udf_probe)
+target_compile_options(udf_probe PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(udf_probe PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ library-yql-minikql
+ udf-service-terminate_policy
+ yql-sql-pg_dummy
+)
+target_link_options(udf_probe PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(udf_probe PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_probe/udf_probe.cpp
+)
+target_allocator(udf_probe
+ system_allocator
+)
+vcs_info(udf_probe)
diff --git a/ydb/library/yql/tools/udf_probe/CMakeLists.linux-aarch64.txt b/ydb/library/yql/tools/udf_probe/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..98f390f523
--- /dev/null
+++ b/ydb/library/yql/tools/udf_probe/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,38 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(udf_probe)
+target_compile_options(udf_probe PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(udf_probe PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-yql-minikql
+ udf-service-terminate_policy
+ yql-sql-pg_dummy
+)
+target_link_options(udf_probe PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(udf_probe PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_probe/udf_probe.cpp
+)
+target_allocator(udf_probe
+ cpp-malloc-jemalloc
+)
+vcs_info(udf_probe)
diff --git a/ydb/library/yql/tools/udf_probe/CMakeLists.linux-x86_64.txt b/ydb/library/yql/tools/udf_probe/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..abc6b25228
--- /dev/null
+++ b/ydb/library/yql/tools/udf_probe/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,40 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(udf_probe)
+target_compile_options(udf_probe PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(udf_probe PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ library-yql-minikql
+ udf-service-terminate_policy
+ yql-sql-pg_dummy
+)
+target_link_options(udf_probe PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(udf_probe PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_probe/udf_probe.cpp
+)
+target_allocator(udf_probe
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(udf_probe)
diff --git a/ydb/library/yql/tools/udf_probe/CMakeLists.txt b/ydb/library/yql/tools/udf_probe/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/ydb/library/yql/tools/udf_probe/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/tools/udf_probe/CMakeLists.windows-x86_64.txt b/ydb/library/yql/tools/udf_probe/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..370590d40e
--- /dev/null
+++ b/ydb/library/yql/tools/udf_probe/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,28 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(udf_probe)
+target_compile_options(udf_probe PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(udf_probe PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ library-yql-minikql
+ udf-service-terminate_policy
+ yql-sql-pg_dummy
+)
+target_sources(udf_probe PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_probe/udf_probe.cpp
+)
+target_allocator(udf_probe
+ system_allocator
+)
+vcs_info(udf_probe)
diff --git a/ydb/library/yql/tools/udf_probe/udf_probe.cpp b/ydb/library/yql/tools/udf_probe/udf_probe.cpp
new file mode 100644
index 0000000000..d47034d243
--- /dev/null
+++ b/ydb/library/yql/tools/udf_probe/udf_probe.cpp
@@ -0,0 +1,35 @@
+#include <ydb/library/yql/minikql/mkql_function_registry.h>
+
+#include <util/generic/yexception.h>
+#include <util/generic/string.h>
+
+using namespace NKikimr;
+using namespace NKikimr::NMiniKQL;
+
+void ListModules(const TString& dir) {
+ TVector<TString> udfPaths;
+ NMiniKQL::FindUdfsInDir(dir, &udfPaths);
+ auto funcRegistry = CreateFunctionRegistry(nullptr, IBuiltinFunctionRegistry::TPtr(), false, udfPaths,
+ NUdf::IRegistrator::TFlags::TypesOnly);
+
+ for (auto& m : funcRegistry->GetAllModuleNames()) {
+ auto path = *funcRegistry->FindUdfPath(m);
+ Cout << m << '\t' << path << Endl;
+ }
+}
+
+int main(int argc, char **argv) {
+ try {
+ if (argc != 2) {
+ Cerr << "Expected directory path\n";
+ }
+
+ TString dir = argv[1];
+ Y_UNUSED(NUdf::GetStaticSymbols());
+ ListModules(dir);
+ return 0;
+ } catch (...) {
+ Cerr << CurrentExceptionMessage() << Endl;
+ return 1;
+ }
+}
diff --git a/ydb/library/yql/tools/udf_probe/ya.make b/ydb/library/yql/tools/udf_probe/ya.make
new file mode 100644
index 0000000000..72e53ba065
--- /dev/null
+++ b/ydb/library/yql/tools/udf_probe/ya.make
@@ -0,0 +1,21 @@
+PROGRAM()
+
+SRCS(
+ udf_probe.cpp
+)
+
+IF (OS_LINUX)
+ # prevent external python extensions to lookup protobuf symbols (and maybe
+ # other common stuff) in main binary
+ EXPORTS_SCRIPT(${ARCADIA_ROOT}/ydb/library/yql/tools/exports.symlist)
+ENDIF()
+
+PEERDIR(
+ ydb/library/yql/minikql
+ ydb/library/yql/public/udf/service/terminate_policy
+ ydb/library/yql/sql/pg_dummy
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/library/yql/tools/udf_resolver/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/tools/udf_resolver/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..553a90185c
--- /dev/null
+++ b/ydb/library/yql/tools/udf_resolver/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,43 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(udf_resolver)
+target_compile_options(udf_resolver PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(udf_resolver PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ library-cpp-getopt
+ cpp-protobuf-util
+ library-yql-minikql
+ udf-service-terminate_policy
+ library-yql-core
+ providers-common-proto
+ common-schema-mkql
+ yql-utils-backtrace
+ yql-utils-sys
+ yql-sql-pg_dummy
+)
+target_link_options(udf_resolver PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(udf_resolver PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_resolver/udf_resolver.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_resolver/discover.cpp
+)
+target_allocator(udf_resolver
+ system_allocator
+)
+vcs_info(udf_resolver)
diff --git a/ydb/library/yql/tools/udf_resolver/CMakeLists.linux-aarch64.txt b/ydb/library/yql/tools/udf_resolver/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..a614caa4ea
--- /dev/null
+++ b/ydb/library/yql/tools/udf_resolver/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,47 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(udf_resolver)
+target_compile_options(udf_resolver PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(udf_resolver PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ contrib-libs-libc_compat
+ library-cpp-getopt
+ cpp-protobuf-util
+ library-yql-minikql
+ udf-service-terminate_policy
+ library-yql-core
+ providers-common-proto
+ common-schema-mkql
+ yql-utils-backtrace
+ yql-utils-sys
+ yql-sql-pg_dummy
+)
+target_link_options(udf_resolver PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(udf_resolver PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_resolver/udf_resolver.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_resolver/discover.cpp
+)
+target_allocator(udf_resolver
+ cpp-malloc-jemalloc
+)
+vcs_info(udf_resolver)
diff --git a/ydb/library/yql/tools/udf_resolver/CMakeLists.linux-x86_64.txt b/ydb/library/yql/tools/udf_resolver/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..c3223efa17
--- /dev/null
+++ b/ydb/library/yql/tools/udf_resolver/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,49 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(udf_resolver)
+target_compile_options(udf_resolver PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(udf_resolver PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ contrib-libs-libc_compat
+ library-cpp-getopt
+ cpp-protobuf-util
+ library-yql-minikql
+ udf-service-terminate_policy
+ library-yql-core
+ providers-common-proto
+ common-schema-mkql
+ yql-utils-backtrace
+ yql-utils-sys
+ yql-sql-pg_dummy
+)
+target_link_options(udf_resolver PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(udf_resolver PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_resolver/udf_resolver.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_resolver/discover.cpp
+)
+target_allocator(udf_resolver
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(udf_resolver)
diff --git a/ydb/library/yql/tools/udf_resolver/CMakeLists.txt b/ydb/library/yql/tools/udf_resolver/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/ydb/library/yql/tools/udf_resolver/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/tools/udf_resolver/CMakeLists.windows-x86_64.txt b/ydb/library/yql/tools/udf_resolver/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..4ee7678263
--- /dev/null
+++ b/ydb/library/yql/tools/udf_resolver/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,36 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(udf_resolver)
+target_compile_options(udf_resolver PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(udf_resolver PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ library-cpp-getopt
+ cpp-protobuf-util
+ library-yql-minikql
+ udf-service-terminate_policy
+ library-yql-core
+ providers-common-proto
+ common-schema-mkql
+ yql-utils-backtrace
+ yql-utils-sys
+ yql-sql-pg_dummy
+)
+target_sources(udf_resolver PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_resolver/udf_resolver.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_resolver/discover.cpp
+)
+target_allocator(udf_resolver
+ system_allocator
+)
+vcs_info(udf_resolver)
diff --git a/ydb/library/yql/tools/udf_resolver/discover.cpp b/ydb/library/yql/tools/udf_resolver/discover.cpp
new file mode 100644
index 0000000000..eb88aed572
--- /dev/null
+++ b/ydb/library/yql/tools/udf_resolver/discover.cpp
@@ -0,0 +1,144 @@
+#include "discover.h"
+
+#include <ydb/library/yql/utils/backtrace/backtrace.h>
+
+#include <ydb/library/yql/minikql/mkql_node.h>
+#include <ydb/library/yql/minikql/mkql_type_builder.h>
+#include <ydb/library/yql/minikql/mkql_utils.h>
+
+#include <ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.h>
+#include <ydb/library/yql/providers/common/proto/udf_resolver.pb.h>
+
+#include <library/cpp/protobuf/util/pb_io.h>
+
+#include <util/generic/hash.h>
+#include <util/generic/hash_set.h>
+
+using namespace NKikimr;
+using namespace NKikimr::NMiniKQL;
+
+namespace NUdfResolver {
+namespace {
+NYql::TResolveResult DoDiscover(const NYql::TResolve& inMsg, IMutableFunctionRegistry& functionRegistry) {
+ NYql::TResolveResult outMsg;
+
+ TScopedAlloc alloc(__LOCATION__);
+ TTypeEnvironment env(alloc);
+
+ NUdf::ITypeInfoHelper::TPtr typeInfoHelper(new TTypeInfoHelper());
+ THashMap<std::pair<TString, TString>, THashSet<TString>> cachedModules;
+ for (auto& import : inMsg.GetImports()) {
+ auto importRes = outMsg.AddImports();
+ importRes->SetFileAlias(import.GetFileAlias());
+ importRes->SetCustomUdfPrefix(import.GetCustomUdfPrefix());
+
+ auto [it, inserted] = cachedModules.emplace(std::make_pair(import.GetPath(), import.GetCustomUdfPrefix()), THashSet<TString>());
+ if (inserted) {
+ THashSet<TString> modules;
+ functionRegistry.LoadUdfs(import.GetPath(),
+ {},
+ NUdf::IRegistrator::TFlags::TypesOnly,
+ import.GetCustomUdfPrefix(),
+ &modules);
+ FillImportResultModules(modules, *importRes);
+ it->second = modules;
+ } else {
+ FillImportResultModules(it->second, *importRes);
+ }
+ }
+
+ for (const auto& module : functionRegistry.GetAllModuleNames()) {
+ const auto& functions = functionRegistry.GetModuleFunctions(module);
+ for (auto& f : functions) {
+ const TString funcName = NKikimr::NMiniKQL::FullName(module, f.first);
+ auto udfRes = outMsg.AddUdfs();
+ udfRes->SetName(funcName);
+ udfRes->SetIsTypeAwareness(f.second.IsTypeAwareness);
+
+ TFunctionTypeInfo funcInfo;
+ if (!f.second.IsTypeAwareness) {
+ auto status = functionRegistry.FindFunctionTypeInfo(env, typeInfoHelper,
+ nullptr, funcName, nullptr, nullptr, NUdf::IUdfModule::TFlags::TypesOnly, {}, nullptr, &funcInfo);
+
+ if (!status.IsOk()) {
+ udfRes->SetError("Failed to resolve signature, error: " + status.GetError());
+ }
+ }
+
+ // nullptr for polymorphic functions
+ if (funcInfo.FunctionType) {
+ udfRes->SetCallableType(NYql::NCommon::WriteTypeToYson(funcInfo.FunctionType, NYT::NYson::EYsonFormat::Text));
+ udfRes->SetArgCount(funcInfo.FunctionType->GetArgumentsCount());
+ udfRes->SetOptionalArgCount(funcInfo.FunctionType->GetOptionalArgumentsCount());
+
+ if (funcInfo.RunConfigType) {
+ udfRes->SetRunConfigType(NYql::NCommon::WriteTypeToYson(funcInfo.RunConfigType, NYT::NYson::EYsonFormat::Text));
+ }
+
+ udfRes->SetSupportsBlocks(funcInfo.SupportsBlocks);
+ udfRes->SetIsStrict(funcInfo.IsStrict);
+ }
+ }
+ }
+
+ return outMsg;
+}
+
+void Print(const NYql::TResolveResult& result, IOutputStream& out, bool printAsProto) {
+ if (printAsProto) {
+ result.SerializeToArcadiaStream(&out);
+ return;
+ }
+
+ SerializeToTextFormat(result, out);
+ out << "UDF count: " << result.UdfsSize() << Endl;
+}
+
+void DiscoverInFiles(const TVector<TString>& udfPaths, IOutputStream& out, bool printAsProto) {
+ NYql::TResolve inMsg;
+ for (auto& path : udfPaths) {
+ auto import = inMsg.AddImports();
+ import->SetPath(path);
+ import->SetFileAlias(path);
+ }
+
+ auto functionRegistry = CreateFunctionRegistry(IBuiltinFunctionRegistry::TPtr());
+ auto newRegistry = functionRegistry->Clone();
+ newRegistry->SetBackTraceCallback(&NYql::NBacktrace::KikimrBackTrace);
+
+ NYql::TResolveResult result = DoDiscover(inMsg, *newRegistry);
+ Print(result, out, printAsProto);
+}
+
+}
+
+void DiscoverInDir(const TString& dir, IOutputStream& out, bool printAsProto) {
+ TVector<TString> udfPaths;
+ NMiniKQL::FindUdfsInDir(dir, &udfPaths);
+ DiscoverInFiles(udfPaths, out, printAsProto);
+}
+
+void DiscoverInFile(const TString& filePath, IOutputStream& out, bool printAsProto) {
+ DiscoverInFiles({ filePath }, out, printAsProto);
+}
+
+void Discover(IInputStream& in, IOutputStream& out, bool printAsProto) {
+ NYql::TResolve inMsg;
+ if (!inMsg.ParseFromArcadiaStream(&in)) {
+ ythrow yexception() << "Bad input TResolve proto message";
+ }
+
+ auto functionRegistry = CreateFunctionRegistry(IBuiltinFunctionRegistry::TPtr());
+ auto newRegistry = functionRegistry->Clone();
+ newRegistry->SetBackTraceCallback(&NYql::NBacktrace::KikimrBackTrace);
+
+ NYql::TResolveResult result = DoDiscover(inMsg, *newRegistry);
+ Print(result, out, printAsProto);
+}
+
+void FillImportResultModules(const THashSet<TString>& modules, NYql::TImportResult& importRes) {
+ for (auto& m : modules) {
+ importRes.AddModules(m);
+ }
+}
+}
diff --git a/ydb/library/yql/tools/udf_resolver/discover.h b/ydb/library/yql/tools/udf_resolver/discover.h
new file mode 100644
index 0000000000..88b92b11ab
--- /dev/null
+++ b/ydb/library/yql/tools/udf_resolver/discover.h
@@ -0,0 +1,16 @@
+#pragma once
+
+#include <ydb/library/yql/providers/common/proto/udf_resolver.pb.h>
+#include <ydb/library/yql/minikql/mkql_function_registry.h>
+
+#include <util/generic/hash.h>
+#include <util/stream/input.h>
+#include <util/stream/output.h>
+
+namespace NUdfResolver {
+void DiscoverInDir(const TString& dir, IOutputStream& out, bool printAsProto);
+void DiscoverInFile(const TString& filePath, IOutputStream& out, bool printAsProto);
+void Discover(IInputStream& in, IOutputStream& out, bool printAsProto);
+void FillImportResultModules(const THashSet<TString>& modules, NYql::TImportResult& importRes);
+;
+}
diff --git a/ydb/library/yql/tools/udf_resolver/udf_resolver.cpp b/ydb/library/yql/tools/udf_resolver/udf_resolver.cpp
new file mode 100644
index 0000000000..0aeb950d48
--- /dev/null
+++ b/ydb/library/yql/tools/udf_resolver/udf_resolver.cpp
@@ -0,0 +1,459 @@
+#include "discover.h"
+
+#include <ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.h>
+#include <ydb/library/yql/providers/common/proto/udf_resolver.pb.h>
+#include <ydb/library/yql/core/yql_type_annotation.h>
+#include <ydb/library/yql/utils/backtrace/backtrace.h>
+#include <ydb/library/yql/utils/sys/become_user.h>
+#include <ydb/library/yql/utils/sys/linux_version.h>
+
+#include <ydb/library/yql/minikql/mkql_function_registry.h>
+#include <ydb/library/yql/minikql/mkql_node.h>
+#include <ydb/library/yql/minikql/mkql_type_builder.h>
+#include <ydb/library/yql/minikql/mkql_program_builder.h>
+#include <ydb/library/yql/minikql/mkql_utils.h>
+
+#include <library/cpp/getopt/last_getopt.h>
+
+#include <util/generic/yexception.h>
+#include <util/generic/string.h>
+#include <util/generic/strbuf.h>
+#include <util/generic/hash_set.h>
+#include <util/generic/hash.h>
+#include <util/system/env.h>
+#include <util/system/fs.h>
+#include <util/stream/output.h>
+#include <util/string/builder.h>
+
+#ifdef _linux_
+#include <sys/types.h>
+#include <sys/prctl.h>
+#include <sys/syscall.h>
+#include <sys/socket.h>
+#include <sys/random.h>
+
+#include <linux/filter.h>
+#include <linux/seccomp.h>
+#include <linux/audit.h>
+#include <linux/random.h>
+
+#ifndef __SI_MAX_SIZE
+#define __SI_MAX_SIZE 128
+#endif
+
+#ifndef __SI_PAD_SIZE
+#if __WORDSIZE == 64
+# define __SI_PAD_SIZE ((__SI_MAX_SIZE / sizeof (int)) - 4)
+#else
+# define __SI_PAD_SIZE ((__SI_MAX_SIZE / sizeof (int)) - 3)
+#endif
+#endif
+
+#endif
+
+using namespace NKikimr;
+using namespace NKikimr::NMiniKQL;
+
+void ResolveUDFs() {
+ NYql::TResolve inMsg;
+ if (!inMsg.ParseFromArcadiaStream(&Cin)) {
+ throw yexception() << "Bad input TResolve proto message";
+ }
+
+ NYql::TResolveResult outMsg;
+
+ auto functionRegistry = CreateFunctionRegistry(IBuiltinFunctionRegistry::TPtr());
+ auto newRegistry = functionRegistry->Clone();
+ newRegistry->SetBackTraceCallback(&NYql::NBacktrace::KikimrBackTrace);
+ TScopedAlloc alloc(__LOCATION__);
+ TTypeEnvironment env(alloc);
+ NUdf::ITypeInfoHelper::TPtr typeInfoHelper(new TTypeInfoHelper);
+
+ TSet<TString> loadedPaths;
+ // Load system imports first
+ for (auto& import : inMsg.GetImports()) {
+ if (import.GetSystem()) {
+ if (!loadedPaths.emplace(import.GetPath()).second) {
+ continue;
+ }
+
+ newRegistry->LoadUdfs(import.GetPath(), {}, NUdf::IRegistrator::TFlags::TypesOnly);
+ }
+ }
+
+ THashMap<std::pair<TString, TString>, THashSet<TString>> cachedModules;
+ for (auto& import : inMsg.GetImports()) {
+ if (!import.GetSystem()) {
+ auto importRes = outMsg.AddImports();
+ importRes->SetFileAlias(import.GetFileAlias());
+ importRes->SetCustomUdfPrefix(import.GetCustomUdfPrefix());
+ auto [it, inserted] = cachedModules.emplace(std::make_pair(import.GetPath(), import.GetCustomUdfPrefix()), THashSet<TString>());
+ if (inserted) {
+ THashSet<TString> modules;
+ newRegistry->LoadUdfs(import.GetPath(),
+ {},
+ NUdf::IRegistrator::TFlags::TypesOnly,
+ import.GetCustomUdfPrefix(),
+ &modules);
+
+ NUdfResolver::FillImportResultModules(modules, *importRes);
+ it->second = modules;
+ } else {
+ NUdfResolver::FillImportResultModules(it->second, *importRes);
+ }
+ }
+ }
+
+ for (size_t i = 0; i < inMsg.UdfsSize(); ++i) {
+ auto& udf = inMsg.GetUdfs(i);
+ auto udfRes = outMsg.AddUdfs();
+ try {
+ TProgramBuilder pgmBuilder(env, *newRegistry);
+ TType* mkqlUserType = nullptr;
+ if (udf.HasUserType()) {
+ TStringStream err;
+ mkqlUserType = NYql::NCommon::ParseTypeFromYson(TStringBuf{udf.GetUserType()}, pgmBuilder, err);
+ if (!mkqlUserType) {
+ udfRes->SetError(TStringBuilder() << "Invalid user type for function: "
+ << udf.GetName() << ", error: " << err.Str());
+ continue;
+ }
+ }
+
+ TFunctionTypeInfo funcInfo;
+ auto status = newRegistry->FindFunctionTypeInfo(env, typeInfoHelper, nullptr,
+ udf.GetName(), mkqlUserType, udf.GetTypeConfig(), NUdf::IUdfModule::TFlags::TypesOnly, {}, nullptr, &funcInfo);
+ if (!status.IsOk()) {
+ udfRes->SetError(TStringBuilder() << "Failed to find UDF function: " << udf.GetName()
+ << ", reason: " << status.GetError());
+ continue;
+ }
+
+ udfRes->SetCallableType(NYql::NCommon::WriteTypeToYson(funcInfo.FunctionType));
+
+ if (funcInfo.RunConfigType) {
+ udfRes->SetRunConfigType(NYql::NCommon::WriteTypeToYson(funcInfo.RunConfigType));
+ }
+
+ if (funcInfo.UserType) {
+ udfRes->SetNormalizedUserType(NYql::NCommon::WriteTypeToYson(funcInfo.UserType));
+ }
+
+ udfRes->SetSupportsBlocks(funcInfo.SupportsBlocks);
+ udfRes->SetIsStrict(funcInfo.IsStrict);
+ } catch (yexception& e) {
+ udfRes->SetError(TStringBuilder()
+ << "Internal error was found when udf metadata is loading for function: " << udf.GetName()
+ << ", reason: " << e.what());
+ }
+ }
+
+ outMsg.SerializeToArcadiaStream(&Cout);
+}
+
+void ListModules(const TString& dir) {
+ TVector<TString> udfPaths;
+ NMiniKQL::FindUdfsInDir(dir, &udfPaths);
+ auto funcRegistry = CreateFunctionRegistry(&NYql::NBacktrace::KikimrBackTrace, IBuiltinFunctionRegistry::TPtr(), false, udfPaths,
+ NUdf::IRegistrator::TFlags::TypesOnly);
+
+ for (auto& m : funcRegistry->GetAllModuleNames()) {
+ auto path = *funcRegistry->FindUdfPath(m);
+ Cout << m << '\t' << path << Endl;
+ }
+}
+
+// NOLINTBEGIN(readability-identifier-naming)
+#ifdef _linux_
+struct my_siginfo_t
+ {
+ int si_signo; /* Signal number. */
+#if __SI_ERRNO_THEN_CODE
+ int si_errno; /* If non-zero, an errno value associated with
+ this signal, as defined in <errno.h>. */
+ int si_code; /* Signal code. */
+#else
+ int si_code;
+ int si_errno;
+#endif
+#if __WORDSIZE == 64
+ int __pad0; /* Explicit padding. */
+#endif
+ union
+ {
+ int _pad[__SI_PAD_SIZE];
+ struct
+ {
+ void *_call_addr; /* Calling user insn. */
+ int _syscall; /* Triggering system call number. */
+ unsigned int _arch; /* AUDIT_ARCH_* of syscall. */
+ } _sigsys;
+
+ } _sifields;
+ };
+// NOLINTEND(readability-identifier-naming)
+
+void SigSysHandler(int sig, my_siginfo_t *info, void *) {
+ Cerr << "SigSysHandler: " << sig << ", code: " << info->si_code << ", errno: " <<
+ info->si_errno << ", call: " << info->_sifields._sigsys._syscall << ", arch:" << info->_sifields._sigsys._arch << "\n";
+ // repeat SIGSYS signal (this will kill current process)
+ raise(sig);
+}
+#endif
+
+int main(int argc, char **argv) {
+ NYql::NBacktrace::RegisterKikimrFatalActions();
+ NYql::NBacktrace::EnableKikimrSymbolize();
+
+ try {
+#ifdef _linux_
+ struct sigaction sa;
+ memset(&sa, 0, sizeof(sa));
+ sa.sa_flags = SA_RESETHAND | SA_SIGINFO;
+ typedef void (*TSigSysHandler)(int, siginfo_t *, void *);
+ sa.sa_sigaction = (TSigSysHandler)SigSysHandler;
+ sigfillset(&sa.sa_mask);
+ if (sigaction(SIGSYS, &sa, nullptr) == -1) {
+ ythrow TSystemError() << "Cannot set handler for signal " << strsignal(SIGSYS);
+ }
+#endif
+
+ TString path;
+ TString user;
+ TString group;
+ bool printAsProto = true;
+
+ NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default();
+ opts.AddLongOption('L', "list", "List UDF modules in specified directory")
+ .Optional()
+ .RequiredArgument("DIR")
+ .StoreResult(&path);
+
+ opts.AddLongOption('D', "discover-path", "Discover UDFs in folder or single file")
+ .Optional()
+ .RequiredArgument("DIR")
+ .StoreResult(&path);
+
+ opts.AddLongOption('P', "discover-proto", "Discover UDFs according to TResolve proto from cin")
+ .Optional()
+ .NoArgument();
+
+ opts.AddLongOption('A', "as-proto", "Print result in protobuf format")
+ .Optional()
+ .DefaultValue(true)
+ .StoreResult(&printAsProto);
+
+ opts.AddLongOption('U', "user", "Run as user")
+ .Optional()
+ .StoreResult(&user);
+
+ opts.AddLongOption('G', "group", "Run as group")
+ .Optional()
+ .StoreResult(&group);
+
+ opts.AddLongOption('F', "filter-syscalls", "Filter syscalls")
+ .Optional()
+ .NoArgument();
+
+ opts.SetFreeArgsNum(0);
+
+ NLastGetopt::TOptsParseResult res(&opts, argc, argv);
+
+ SetEnv("USER", user ? user : "udf_resolver");
+ if (user && !NYql::IsLinuxKernelBelow4_3()) {
+ // since we are going to load untrusted modules
+ // we have to switch into another user and drop privileges (capabilities)
+ NYql::BecomeUser(user, group);
+ NYql::TurnOffBecomeUserAbility();
+ }
+
+ NYql::SendSignalOnParentThreadExit(SIGTERM);
+
+ if (res.Has("filter-syscalls")) {
+#ifdef _linux_
+
+#define ArchField offsetof(struct seccomp_data, arch) // NOLINT(readability-identifier-naming)
+
+// NOLINTNEXTLINE(readability-identifier-naming)
+#define Allow(syscall) \
+ BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, SYS_##syscall, 0, 1), \
+ BPF_STMT(BPF_RET+BPF_K, SECCOMP_RET_ALLOW)
+
+ struct sock_filter filter[] = {
+ /* validate arch */
+ BPF_STMT(BPF_LD+BPF_W+BPF_ABS, ArchField),
+ BPF_JUMP( BPF_JMP+BPF_JEQ+BPF_K, AUDIT_ARCH_X86_64, 1, 0),
+ BPF_STMT(BPF_RET+BPF_K, SECCOMP_RET_TRAP),
+
+ /* load syscall */
+ BPF_STMT(BPF_LD+BPF_W+BPF_ABS, offsetof(struct seccomp_data, nr)),
+
+ /* list of allowed syscalls */
+#ifndef _arm64_
+ Allow(access),
+#endif
+ Allow(brk),
+ Allow(chdir),
+ Allow(clock_gettime),
+ Allow(clone),
+ Allow(close),
+#ifndef _arm64_
+ Allow(creat),
+#endif
+ Allow(dup),
+#ifndef _arm64_
+ Allow(dup2),
+#endif
+ Allow(dup3),
+ Allow(eventfd2),
+ Allow(exit),
+ Allow(exit_group),
+ Allow(fadvise64),
+ Allow(fallocate),
+ Allow(flock),
+ Allow(fstat),
+ Allow(fsync),
+ Allow(ftruncate),
+ Allow(futex),
+ Allow(get_robust_list),
+ Allow(getcwd),
+#ifndef _arm64_
+ Allow(getdents),
+#endif
+ Allow(getdents64),
+ Allow(getegid),
+ Allow(geteuid),
+ Allow(getgid),
+ Allow(getgroups),
+ Allow(getpgid),
+#ifndef _arm64_
+ Allow(getpgrp),
+#endif
+ Allow(getpid),
+ Allow(getppid),
+ Allow(getpriority),
+ Allow(getrandom),
+ Allow(getrlimit),
+ Allow(getrusage),
+ Allow(getsid),
+ Allow(gettid),
+ Allow(gettimeofday),
+ Allow(getuid),
+ Allow(getxattr),
+ Allow(ioctl),
+ Allow(lgetxattr),
+#ifndef _arm64_
+ Allow(link),
+#endif
+ Allow(listxattr),
+ Allow(llistxattr),
+ Allow(lremovexattr),
+ Allow(lseek),
+ Allow(lsetxattr),
+#ifndef _arm64_
+ Allow(lstat),
+#endif
+ Allow(madvise),
+#ifndef _arm64_
+ Allow(mkdir),
+#endif
+ Allow(mkdirat),
+ Allow(mlock),
+ Allow(mlockall),
+ Allow(mmap),
+ Allow(mprotect),
+ Allow(munlock),
+ Allow(munlockall),
+ Allow(munmap),
+ Allow(nanosleep),
+#ifndef _arm64_
+ Allow(open),
+#endif
+ Allow(openat),
+ Allow(pipe2),
+ Allow(prctl),
+ Allow(pread64),
+ Allow(pwrite64),
+ Allow(read),
+#ifndef _arm64_
+ Allow(readlink),
+#endif
+ Allow(readv),
+ Allow(removexattr),
+#ifndef _arm64_
+ Allow(rename),
+#endif
+ Allow(renameat),
+#ifndef _arm64_
+ Allow(rmdir),
+#endif
+ Allow(rt_sigaction),
+ Allow(rt_sigpending),
+ Allow(rt_sigprocmask),
+ Allow(rt_sigqueueinfo),
+ Allow(rt_sigreturn),
+ Allow(rt_sigsuspend),
+ Allow(rt_sigtimedwait),
+ Allow(rt_tgsigqueueinfo),
+ Allow(sched_getaffinity),
+ Allow(sched_setaffinity),
+ Allow(set_robust_list),
+ Allow(setxattr),
+#ifndef _arm64_
+ Allow(stat),
+#endif
+ Allow(sysinfo),
+ Allow(uname),
+#ifndef _arm64_
+ Allow(unlink),
+#endif
+ Allow(unlinkat),
+ Allow(write),
+ Allow(writev),
+
+ /* and if we don't match above, die */
+ BPF_STMT(BPF_RET+BPF_K, SECCOMP_RET_TRAP),
+ };
+ struct sock_fprog filterprog = {
+ .len = sizeof(filter)/sizeof(filter[0]),
+ .filter = filter
+ };
+
+ if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) == -1) {
+ ythrow yexception() << "prctl(PR_SET_NO_NEW_PRIVS, 1, ...) failed with: " << LastSystemErrorText();
+ }
+ if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &filterprog) == -1) {
+ ythrow yexception() << "prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, ...) failed with: " << LastSystemErrorText();
+ }
+#endif
+ }
+
+ Y_UNUSED(NUdf::GetStaticSymbols());
+ if (res.Has("list")) {
+ ListModules(path);
+ return 0;
+ }
+
+ if (res.Has("discover-path")) {
+ NFs::EnsureExists(path);
+ TFileStat fstat(path);
+ if (fstat.IsDir()) {
+ NUdfResolver::DiscoverInDir(path, Cout, printAsProto);
+ } else {
+ NUdfResolver::DiscoverInFile(path, Cout, printAsProto);
+ }
+ return 0;
+ }
+
+ if (res.Has("discover-proto")) {
+ NUdfResolver::Discover(Cin, Cout, printAsProto);
+ return 0;
+ }
+
+ ResolveUDFs();
+ return 0;
+ } catch (...) {
+ Cerr << CurrentExceptionMessage() << Endl;
+ return 1;
+ }
+}
diff --git a/ydb/library/yql/tools/udf_resolver/ya.make b/ydb/library/yql/tools/udf_resolver/ya.make
new file mode 100644
index 0000000000..71a8344e19
--- /dev/null
+++ b/ydb/library/yql/tools/udf_resolver/ya.make
@@ -0,0 +1,33 @@
+PROGRAM()
+
+SRCS(
+ udf_resolver.cpp
+ discover.cpp
+)
+
+IF (OS_LINUX)
+ # prevent external python extensions to lookup protobuf symbols (and maybe
+ # other common stuff) in main binary
+ EXPORTS_SCRIPT(${ARCADIA_ROOT}/ydb/library/yql/tools/exports.symlist)
+
+ PEERDIR(
+ contrib/libs/libc_compat
+ )
+ENDIF()
+
+PEERDIR(
+ library/cpp/getopt
+ library/cpp/protobuf/util
+ ydb/library/yql/minikql
+ ydb/library/yql/public/udf/service/terminate_policy
+ ydb/library/yql/core
+ ydb/library/yql/providers/common/proto
+ ydb/library/yql/providers/common/schema/mkql
+ ydb/library/yql/utils/backtrace
+ ydb/library/yql/utils/sys
+ ydb/library/yql/sql/pg_dummy
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/library/yql/tools/ya.make b/ydb/library/yql/tools/ya.make
index 5ff8e2b273..4631e3b04e 100644
--- a/ydb/library/yql/tools/ya.make
+++ b/ydb/library/yql/tools/ya.make
@@ -5,5 +5,8 @@ RECURSE(
mrjob
sql2yql
sql_formatter
+ udf_dep_stub
+ udf_probe
+ udf_resolver
yqlrun
)