diff options
author | vvvv <vvvv@ydb.tech> | 2023-11-12 14:17:41 +0300 |
---|---|---|
committer | vvvv <vvvv@ydb.tech> | 2023-11-12 14:32:51 +0300 |
commit | e4bf7da8cc01e5fd68c54a0d299c2d07c67b0076 (patch) | |
tree | 5ddd0290b679c00cbf6a450fe31db60c167349cc | |
parent | 33a303b290c6dec1b122a3c782f0b17fc6ea6413 (diff) | |
download | ydb-e4bf7da8cc01e5fd68c54a0d299c2d07c67b0076.tar.gz |
YQL-16964 clone udf tools into ydb
26 files changed, 1206 insertions, 0 deletions
diff --git a/.mapping.json b/.mapping.json index b31a969bf0..5c586a131e 100644 --- a/.mapping.json +++ b/.mapping.json @@ -8475,6 +8475,21 @@ "ydb/library/yql/tools/sql_formatter/CMakeLists.linux-x86_64.txt":"", "ydb/library/yql/tools/sql_formatter/CMakeLists.txt":"", "ydb/library/yql/tools/sql_formatter/CMakeLists.windows-x86_64.txt":"", + "ydb/library/yql/tools/udf_dep_stub/CMakeLists.darwin-x86_64.txt":"", + "ydb/library/yql/tools/udf_dep_stub/CMakeLists.linux-aarch64.txt":"", + "ydb/library/yql/tools/udf_dep_stub/CMakeLists.linux-x86_64.txt":"", + "ydb/library/yql/tools/udf_dep_stub/CMakeLists.txt":"", + "ydb/library/yql/tools/udf_dep_stub/CMakeLists.windows-x86_64.txt":"", + "ydb/library/yql/tools/udf_probe/CMakeLists.darwin-x86_64.txt":"", + "ydb/library/yql/tools/udf_probe/CMakeLists.linux-aarch64.txt":"", + "ydb/library/yql/tools/udf_probe/CMakeLists.linux-x86_64.txt":"", + "ydb/library/yql/tools/udf_probe/CMakeLists.txt":"", + "ydb/library/yql/tools/udf_probe/CMakeLists.windows-x86_64.txt":"", + "ydb/library/yql/tools/udf_resolver/CMakeLists.darwin-x86_64.txt":"", + "ydb/library/yql/tools/udf_resolver/CMakeLists.linux-aarch64.txt":"", + "ydb/library/yql/tools/udf_resolver/CMakeLists.linux-x86_64.txt":"", + "ydb/library/yql/tools/udf_resolver/CMakeLists.txt":"", + "ydb/library/yql/tools/udf_resolver/CMakeLists.windows-x86_64.txt":"", "ydb/library/yql/tools/yqlrun/CMakeLists.darwin-x86_64.txt":"", "ydb/library/yql/tools/yqlrun/CMakeLists.linux-aarch64.txt":"", "ydb/library/yql/tools/yqlrun/CMakeLists.linux-x86_64.txt":"", diff --git a/ydb/library/yql/tools/CMakeLists.txt b/ydb/library/yql/tools/CMakeLists.txt index 61ab79236b..1d7c5f669c 100644 --- a/ydb/library/yql/tools/CMakeLists.txt +++ b/ydb/library/yql/tools/CMakeLists.txt @@ -12,4 +12,7 @@ add_subdirectory(dqrun) add_subdirectory(mrjob) add_subdirectory(sql2yql) add_subdirectory(sql_formatter) +add_subdirectory(udf_dep_stub) +add_subdirectory(udf_probe) +add_subdirectory(udf_resolver) add_subdirectory(yqlrun) diff --git a/ydb/library/yql/tools/udf_dep_stub/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..6e0b1af6b7 --- /dev/null +++ b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,25 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_shared_library(yql_udf_dep_stub) +target_link_libraries(yql_udf_dep_stub PUBLIC + contrib-libs-cxxsupp + yutil +) +target_link_options(yql_udf_dep_stub PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -undefined + dynamic_lookup + -fPIC +) +target_sources(yql_udf_dep_stub PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_dep_stub/main.cpp +) +vcs_info(yql_udf_dep_stub) diff --git a/ydb/library/yql/tools/udf_dep_stub/CMakeLists.linux-aarch64.txt b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..4cc94a170c --- /dev/null +++ b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.linux-aarch64.txt @@ -0,0 +1,30 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_shared_library(yql_udf_dep_stub) +target_link_libraries(yql_udf_dep_stub PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil +) +target_link_options(yql_udf_dep_stub PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -Wl,-z,notext + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(yql_udf_dep_stub PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_dep_stub/main.cpp +) +vcs_info(yql_udf_dep_stub) diff --git a/ydb/library/yql/tools/udf_dep_stub/CMakeLists.linux-x86_64.txt b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..4cc94a170c --- /dev/null +++ b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.linux-x86_64.txt @@ -0,0 +1,30 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_shared_library(yql_udf_dep_stub) +target_link_libraries(yql_udf_dep_stub PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil +) +target_link_options(yql_udf_dep_stub PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -Wl,-z,notext + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(yql_udf_dep_stub PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_dep_stub/main.cpp +) +vcs_info(yql_udf_dep_stub) diff --git a/ydb/library/yql/tools/udf_dep_stub/CMakeLists.txt b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/tools/udf_dep_stub/CMakeLists.windows-x86_64.txt b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..17efeeeaf5 --- /dev/null +++ b/ydb/library/yql/tools/udf_dep_stub/CMakeLists.windows-x86_64.txt @@ -0,0 +1,18 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_shared_library(yql_udf_dep_stub) +target_link_libraries(yql_udf_dep_stub PUBLIC + contrib-libs-cxxsupp + yutil +) +target_sources(yql_udf_dep_stub PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_dep_stub/main.cpp +) +vcs_info(yql_udf_dep_stub) diff --git a/ydb/library/yql/tools/udf_dep_stub/main.cpp b/ydb/library/yql/tools/udf_dep_stub/main.cpp new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/ydb/library/yql/tools/udf_dep_stub/main.cpp diff --git a/ydb/library/yql/tools/udf_dep_stub/ya.make b/ydb/library/yql/tools/udf_dep_stub/ya.make new file mode 100644 index 0000000000..abe778ed12 --- /dev/null +++ b/ydb/library/yql/tools/udf_dep_stub/ya.make @@ -0,0 +1,7 @@ +DLL(yql_udf_dep_stub) + +SRCS( + main.cpp +) + +END() diff --git a/ydb/library/yql/tools/udf_probe/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/tools/udf_probe/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..1edbd44aeb --- /dev/null +++ b/ydb/library/yql/tools/udf_probe/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,35 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(udf_probe) +target_compile_options(udf_probe PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(udf_probe PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + library-yql-minikql + udf-service-terminate_policy + yql-sql-pg_dummy +) +target_link_options(udf_probe PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(udf_probe PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_probe/udf_probe.cpp +) +target_allocator(udf_probe + system_allocator +) +vcs_info(udf_probe) diff --git a/ydb/library/yql/tools/udf_probe/CMakeLists.linux-aarch64.txt b/ydb/library/yql/tools/udf_probe/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..98f390f523 --- /dev/null +++ b/ydb/library/yql/tools/udf_probe/CMakeLists.linux-aarch64.txt @@ -0,0 +1,38 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(udf_probe) +target_compile_options(udf_probe PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(udf_probe PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-yql-minikql + udf-service-terminate_policy + yql-sql-pg_dummy +) +target_link_options(udf_probe PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(udf_probe PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_probe/udf_probe.cpp +) +target_allocator(udf_probe + cpp-malloc-jemalloc +) +vcs_info(udf_probe) diff --git a/ydb/library/yql/tools/udf_probe/CMakeLists.linux-x86_64.txt b/ydb/library/yql/tools/udf_probe/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..abc6b25228 --- /dev/null +++ b/ydb/library/yql/tools/udf_probe/CMakeLists.linux-x86_64.txt @@ -0,0 +1,40 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(udf_probe) +target_compile_options(udf_probe PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(udf_probe PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + library-yql-minikql + udf-service-terminate_policy + yql-sql-pg_dummy +) +target_link_options(udf_probe PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(udf_probe PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_probe/udf_probe.cpp +) +target_allocator(udf_probe + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(udf_probe) diff --git a/ydb/library/yql/tools/udf_probe/CMakeLists.txt b/ydb/library/yql/tools/udf_probe/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/library/yql/tools/udf_probe/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/tools/udf_probe/CMakeLists.windows-x86_64.txt b/ydb/library/yql/tools/udf_probe/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..370590d40e --- /dev/null +++ b/ydb/library/yql/tools/udf_probe/CMakeLists.windows-x86_64.txt @@ -0,0 +1,28 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(udf_probe) +target_compile_options(udf_probe PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(udf_probe PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + library-yql-minikql + udf-service-terminate_policy + yql-sql-pg_dummy +) +target_sources(udf_probe PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_probe/udf_probe.cpp +) +target_allocator(udf_probe + system_allocator +) +vcs_info(udf_probe) diff --git a/ydb/library/yql/tools/udf_probe/udf_probe.cpp b/ydb/library/yql/tools/udf_probe/udf_probe.cpp new file mode 100644 index 0000000000..d47034d243 --- /dev/null +++ b/ydb/library/yql/tools/udf_probe/udf_probe.cpp @@ -0,0 +1,35 @@ +#include <ydb/library/yql/minikql/mkql_function_registry.h> + +#include <util/generic/yexception.h> +#include <util/generic/string.h> + +using namespace NKikimr; +using namespace NKikimr::NMiniKQL; + +void ListModules(const TString& dir) { + TVector<TString> udfPaths; + NMiniKQL::FindUdfsInDir(dir, &udfPaths); + auto funcRegistry = CreateFunctionRegistry(nullptr, IBuiltinFunctionRegistry::TPtr(), false, udfPaths, + NUdf::IRegistrator::TFlags::TypesOnly); + + for (auto& m : funcRegistry->GetAllModuleNames()) { + auto path = *funcRegistry->FindUdfPath(m); + Cout << m << '\t' << path << Endl; + } +} + +int main(int argc, char **argv) { + try { + if (argc != 2) { + Cerr << "Expected directory path\n"; + } + + TString dir = argv[1]; + Y_UNUSED(NUdf::GetStaticSymbols()); + ListModules(dir); + return 0; + } catch (...) { + Cerr << CurrentExceptionMessage() << Endl; + return 1; + } +} diff --git a/ydb/library/yql/tools/udf_probe/ya.make b/ydb/library/yql/tools/udf_probe/ya.make new file mode 100644 index 0000000000..72e53ba065 --- /dev/null +++ b/ydb/library/yql/tools/udf_probe/ya.make @@ -0,0 +1,21 @@ +PROGRAM() + +SRCS( + udf_probe.cpp +) + +IF (OS_LINUX) + # prevent external python extensions to lookup protobuf symbols (and maybe + # other common stuff) in main binary + EXPORTS_SCRIPT(${ARCADIA_ROOT}/ydb/library/yql/tools/exports.symlist) +ENDIF() + +PEERDIR( + ydb/library/yql/minikql + ydb/library/yql/public/udf/service/terminate_policy + ydb/library/yql/sql/pg_dummy +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/library/yql/tools/udf_resolver/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/tools/udf_resolver/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..553a90185c --- /dev/null +++ b/ydb/library/yql/tools/udf_resolver/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,43 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(udf_resolver) +target_compile_options(udf_resolver PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(udf_resolver PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + library-cpp-getopt + cpp-protobuf-util + library-yql-minikql + udf-service-terminate_policy + library-yql-core + providers-common-proto + common-schema-mkql + yql-utils-backtrace + yql-utils-sys + yql-sql-pg_dummy +) +target_link_options(udf_resolver PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(udf_resolver PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_resolver/udf_resolver.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_resolver/discover.cpp +) +target_allocator(udf_resolver + system_allocator +) +vcs_info(udf_resolver) diff --git a/ydb/library/yql/tools/udf_resolver/CMakeLists.linux-aarch64.txt b/ydb/library/yql/tools/udf_resolver/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..a614caa4ea --- /dev/null +++ b/ydb/library/yql/tools/udf_resolver/CMakeLists.linux-aarch64.txt @@ -0,0 +1,47 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(udf_resolver) +target_compile_options(udf_resolver PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(udf_resolver PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + contrib-libs-libc_compat + library-cpp-getopt + cpp-protobuf-util + library-yql-minikql + udf-service-terminate_policy + library-yql-core + providers-common-proto + common-schema-mkql + yql-utils-backtrace + yql-utils-sys + yql-sql-pg_dummy +) +target_link_options(udf_resolver PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(udf_resolver PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_resolver/udf_resolver.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_resolver/discover.cpp +) +target_allocator(udf_resolver + cpp-malloc-jemalloc +) +vcs_info(udf_resolver) diff --git a/ydb/library/yql/tools/udf_resolver/CMakeLists.linux-x86_64.txt b/ydb/library/yql/tools/udf_resolver/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..c3223efa17 --- /dev/null +++ b/ydb/library/yql/tools/udf_resolver/CMakeLists.linux-x86_64.txt @@ -0,0 +1,49 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(udf_resolver) +target_compile_options(udf_resolver PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(udf_resolver PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + contrib-libs-libc_compat + library-cpp-getopt + cpp-protobuf-util + library-yql-minikql + udf-service-terminate_policy + library-yql-core + providers-common-proto + common-schema-mkql + yql-utils-backtrace + yql-utils-sys + yql-sql-pg_dummy +) +target_link_options(udf_resolver PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(udf_resolver PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_resolver/udf_resolver.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_resolver/discover.cpp +) +target_allocator(udf_resolver + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(udf_resolver) diff --git a/ydb/library/yql/tools/udf_resolver/CMakeLists.txt b/ydb/library/yql/tools/udf_resolver/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/library/yql/tools/udf_resolver/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/tools/udf_resolver/CMakeLists.windows-x86_64.txt b/ydb/library/yql/tools/udf_resolver/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..4ee7678263 --- /dev/null +++ b/ydb/library/yql/tools/udf_resolver/CMakeLists.windows-x86_64.txt @@ -0,0 +1,36 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(udf_resolver) +target_compile_options(udf_resolver PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(udf_resolver PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + library-cpp-getopt + cpp-protobuf-util + library-yql-minikql + udf-service-terminate_policy + library-yql-core + providers-common-proto + common-schema-mkql + yql-utils-backtrace + yql-utils-sys + yql-sql-pg_dummy +) +target_sources(udf_resolver PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_resolver/udf_resolver.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/udf_resolver/discover.cpp +) +target_allocator(udf_resolver + system_allocator +) +vcs_info(udf_resolver) diff --git a/ydb/library/yql/tools/udf_resolver/discover.cpp b/ydb/library/yql/tools/udf_resolver/discover.cpp new file mode 100644 index 0000000000..eb88aed572 --- /dev/null +++ b/ydb/library/yql/tools/udf_resolver/discover.cpp @@ -0,0 +1,144 @@ +#include "discover.h" + +#include <ydb/library/yql/utils/backtrace/backtrace.h> + +#include <ydb/library/yql/minikql/mkql_node.h> +#include <ydb/library/yql/minikql/mkql_type_builder.h> +#include <ydb/library/yql/minikql/mkql_utils.h> + +#include <ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.h> +#include <ydb/library/yql/providers/common/proto/udf_resolver.pb.h> + +#include <library/cpp/protobuf/util/pb_io.h> + +#include <util/generic/hash.h> +#include <util/generic/hash_set.h> + +using namespace NKikimr; +using namespace NKikimr::NMiniKQL; + +namespace NUdfResolver { +namespace { +NYql::TResolveResult DoDiscover(const NYql::TResolve& inMsg, IMutableFunctionRegistry& functionRegistry) { + NYql::TResolveResult outMsg; + + TScopedAlloc alloc(__LOCATION__); + TTypeEnvironment env(alloc); + + NUdf::ITypeInfoHelper::TPtr typeInfoHelper(new TTypeInfoHelper()); + THashMap<std::pair<TString, TString>, THashSet<TString>> cachedModules; + for (auto& import : inMsg.GetImports()) { + auto importRes = outMsg.AddImports(); + importRes->SetFileAlias(import.GetFileAlias()); + importRes->SetCustomUdfPrefix(import.GetCustomUdfPrefix()); + + auto [it, inserted] = cachedModules.emplace(std::make_pair(import.GetPath(), import.GetCustomUdfPrefix()), THashSet<TString>()); + if (inserted) { + THashSet<TString> modules; + functionRegistry.LoadUdfs(import.GetPath(), + {}, + NUdf::IRegistrator::TFlags::TypesOnly, + import.GetCustomUdfPrefix(), + &modules); + FillImportResultModules(modules, *importRes); + it->second = modules; + } else { + FillImportResultModules(it->second, *importRes); + } + } + + for (const auto& module : functionRegistry.GetAllModuleNames()) { + const auto& functions = functionRegistry.GetModuleFunctions(module); + for (auto& f : functions) { + const TString funcName = NKikimr::NMiniKQL::FullName(module, f.first); + auto udfRes = outMsg.AddUdfs(); + udfRes->SetName(funcName); + udfRes->SetIsTypeAwareness(f.second.IsTypeAwareness); + + TFunctionTypeInfo funcInfo; + if (!f.second.IsTypeAwareness) { + auto status = functionRegistry.FindFunctionTypeInfo(env, typeInfoHelper, + nullptr, funcName, nullptr, nullptr, NUdf::IUdfModule::TFlags::TypesOnly, {}, nullptr, &funcInfo); + + if (!status.IsOk()) { + udfRes->SetError("Failed to resolve signature, error: " + status.GetError()); + } + } + + // nullptr for polymorphic functions + if (funcInfo.FunctionType) { + udfRes->SetCallableType(NYql::NCommon::WriteTypeToYson(funcInfo.FunctionType, NYT::NYson::EYsonFormat::Text)); + udfRes->SetArgCount(funcInfo.FunctionType->GetArgumentsCount()); + udfRes->SetOptionalArgCount(funcInfo.FunctionType->GetOptionalArgumentsCount()); + + if (funcInfo.RunConfigType) { + udfRes->SetRunConfigType(NYql::NCommon::WriteTypeToYson(funcInfo.RunConfigType, NYT::NYson::EYsonFormat::Text)); + } + + udfRes->SetSupportsBlocks(funcInfo.SupportsBlocks); + udfRes->SetIsStrict(funcInfo.IsStrict); + } + } + } + + return outMsg; +} + +void Print(const NYql::TResolveResult& result, IOutputStream& out, bool printAsProto) { + if (printAsProto) { + result.SerializeToArcadiaStream(&out); + return; + } + + SerializeToTextFormat(result, out); + out << "UDF count: " << result.UdfsSize() << Endl; +} + +void DiscoverInFiles(const TVector<TString>& udfPaths, IOutputStream& out, bool printAsProto) { + NYql::TResolve inMsg; + for (auto& path : udfPaths) { + auto import = inMsg.AddImports(); + import->SetPath(path); + import->SetFileAlias(path); + } + + auto functionRegistry = CreateFunctionRegistry(IBuiltinFunctionRegistry::TPtr()); + auto newRegistry = functionRegistry->Clone(); + newRegistry->SetBackTraceCallback(&NYql::NBacktrace::KikimrBackTrace); + + NYql::TResolveResult result = DoDiscover(inMsg, *newRegistry); + Print(result, out, printAsProto); +} + +} + +void DiscoverInDir(const TString& dir, IOutputStream& out, bool printAsProto) { + TVector<TString> udfPaths; + NMiniKQL::FindUdfsInDir(dir, &udfPaths); + DiscoverInFiles(udfPaths, out, printAsProto); +} + +void DiscoverInFile(const TString& filePath, IOutputStream& out, bool printAsProto) { + DiscoverInFiles({ filePath }, out, printAsProto); +} + +void Discover(IInputStream& in, IOutputStream& out, bool printAsProto) { + NYql::TResolve inMsg; + if (!inMsg.ParseFromArcadiaStream(&in)) { + ythrow yexception() << "Bad input TResolve proto message"; + } + + auto functionRegistry = CreateFunctionRegistry(IBuiltinFunctionRegistry::TPtr()); + auto newRegistry = functionRegistry->Clone(); + newRegistry->SetBackTraceCallback(&NYql::NBacktrace::KikimrBackTrace); + + NYql::TResolveResult result = DoDiscover(inMsg, *newRegistry); + Print(result, out, printAsProto); +} + +void FillImportResultModules(const THashSet<TString>& modules, NYql::TImportResult& importRes) { + for (auto& m : modules) { + importRes.AddModules(m); + } +} +} diff --git a/ydb/library/yql/tools/udf_resolver/discover.h b/ydb/library/yql/tools/udf_resolver/discover.h new file mode 100644 index 0000000000..88b92b11ab --- /dev/null +++ b/ydb/library/yql/tools/udf_resolver/discover.h @@ -0,0 +1,16 @@ +#pragma once + +#include <ydb/library/yql/providers/common/proto/udf_resolver.pb.h> +#include <ydb/library/yql/minikql/mkql_function_registry.h> + +#include <util/generic/hash.h> +#include <util/stream/input.h> +#include <util/stream/output.h> + +namespace NUdfResolver { +void DiscoverInDir(const TString& dir, IOutputStream& out, bool printAsProto); +void DiscoverInFile(const TString& filePath, IOutputStream& out, bool printAsProto); +void Discover(IInputStream& in, IOutputStream& out, bool printAsProto); +void FillImportResultModules(const THashSet<TString>& modules, NYql::TImportResult& importRes); +; +} diff --git a/ydb/library/yql/tools/udf_resolver/udf_resolver.cpp b/ydb/library/yql/tools/udf_resolver/udf_resolver.cpp new file mode 100644 index 0000000000..0aeb950d48 --- /dev/null +++ b/ydb/library/yql/tools/udf_resolver/udf_resolver.cpp @@ -0,0 +1,459 @@ +#include "discover.h" + +#include <ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.h> +#include <ydb/library/yql/providers/common/proto/udf_resolver.pb.h> +#include <ydb/library/yql/core/yql_type_annotation.h> +#include <ydb/library/yql/utils/backtrace/backtrace.h> +#include <ydb/library/yql/utils/sys/become_user.h> +#include <ydb/library/yql/utils/sys/linux_version.h> + +#include <ydb/library/yql/minikql/mkql_function_registry.h> +#include <ydb/library/yql/minikql/mkql_node.h> +#include <ydb/library/yql/minikql/mkql_type_builder.h> +#include <ydb/library/yql/minikql/mkql_program_builder.h> +#include <ydb/library/yql/minikql/mkql_utils.h> + +#include <library/cpp/getopt/last_getopt.h> + +#include <util/generic/yexception.h> +#include <util/generic/string.h> +#include <util/generic/strbuf.h> +#include <util/generic/hash_set.h> +#include <util/generic/hash.h> +#include <util/system/env.h> +#include <util/system/fs.h> +#include <util/stream/output.h> +#include <util/string/builder.h> + +#ifdef _linux_ +#include <sys/types.h> +#include <sys/prctl.h> +#include <sys/syscall.h> +#include <sys/socket.h> +#include <sys/random.h> + +#include <linux/filter.h> +#include <linux/seccomp.h> +#include <linux/audit.h> +#include <linux/random.h> + +#ifndef __SI_MAX_SIZE +#define __SI_MAX_SIZE 128 +#endif + +#ifndef __SI_PAD_SIZE +#if __WORDSIZE == 64 +# define __SI_PAD_SIZE ((__SI_MAX_SIZE / sizeof (int)) - 4) +#else +# define __SI_PAD_SIZE ((__SI_MAX_SIZE / sizeof (int)) - 3) +#endif +#endif + +#endif + +using namespace NKikimr; +using namespace NKikimr::NMiniKQL; + +void ResolveUDFs() { + NYql::TResolve inMsg; + if (!inMsg.ParseFromArcadiaStream(&Cin)) { + throw yexception() << "Bad input TResolve proto message"; + } + + NYql::TResolveResult outMsg; + + auto functionRegistry = CreateFunctionRegistry(IBuiltinFunctionRegistry::TPtr()); + auto newRegistry = functionRegistry->Clone(); + newRegistry->SetBackTraceCallback(&NYql::NBacktrace::KikimrBackTrace); + TScopedAlloc alloc(__LOCATION__); + TTypeEnvironment env(alloc); + NUdf::ITypeInfoHelper::TPtr typeInfoHelper(new TTypeInfoHelper); + + TSet<TString> loadedPaths; + // Load system imports first + for (auto& import : inMsg.GetImports()) { + if (import.GetSystem()) { + if (!loadedPaths.emplace(import.GetPath()).second) { + continue; + } + + newRegistry->LoadUdfs(import.GetPath(), {}, NUdf::IRegistrator::TFlags::TypesOnly); + } + } + + THashMap<std::pair<TString, TString>, THashSet<TString>> cachedModules; + for (auto& import : inMsg.GetImports()) { + if (!import.GetSystem()) { + auto importRes = outMsg.AddImports(); + importRes->SetFileAlias(import.GetFileAlias()); + importRes->SetCustomUdfPrefix(import.GetCustomUdfPrefix()); + auto [it, inserted] = cachedModules.emplace(std::make_pair(import.GetPath(), import.GetCustomUdfPrefix()), THashSet<TString>()); + if (inserted) { + THashSet<TString> modules; + newRegistry->LoadUdfs(import.GetPath(), + {}, + NUdf::IRegistrator::TFlags::TypesOnly, + import.GetCustomUdfPrefix(), + &modules); + + NUdfResolver::FillImportResultModules(modules, *importRes); + it->second = modules; + } else { + NUdfResolver::FillImportResultModules(it->second, *importRes); + } + } + } + + for (size_t i = 0; i < inMsg.UdfsSize(); ++i) { + auto& udf = inMsg.GetUdfs(i); + auto udfRes = outMsg.AddUdfs(); + try { + TProgramBuilder pgmBuilder(env, *newRegistry); + TType* mkqlUserType = nullptr; + if (udf.HasUserType()) { + TStringStream err; + mkqlUserType = NYql::NCommon::ParseTypeFromYson(TStringBuf{udf.GetUserType()}, pgmBuilder, err); + if (!mkqlUserType) { + udfRes->SetError(TStringBuilder() << "Invalid user type for function: " + << udf.GetName() << ", error: " << err.Str()); + continue; + } + } + + TFunctionTypeInfo funcInfo; + auto status = newRegistry->FindFunctionTypeInfo(env, typeInfoHelper, nullptr, + udf.GetName(), mkqlUserType, udf.GetTypeConfig(), NUdf::IUdfModule::TFlags::TypesOnly, {}, nullptr, &funcInfo); + if (!status.IsOk()) { + udfRes->SetError(TStringBuilder() << "Failed to find UDF function: " << udf.GetName() + << ", reason: " << status.GetError()); + continue; + } + + udfRes->SetCallableType(NYql::NCommon::WriteTypeToYson(funcInfo.FunctionType)); + + if (funcInfo.RunConfigType) { + udfRes->SetRunConfigType(NYql::NCommon::WriteTypeToYson(funcInfo.RunConfigType)); + } + + if (funcInfo.UserType) { + udfRes->SetNormalizedUserType(NYql::NCommon::WriteTypeToYson(funcInfo.UserType)); + } + + udfRes->SetSupportsBlocks(funcInfo.SupportsBlocks); + udfRes->SetIsStrict(funcInfo.IsStrict); + } catch (yexception& e) { + udfRes->SetError(TStringBuilder() + << "Internal error was found when udf metadata is loading for function: " << udf.GetName() + << ", reason: " << e.what()); + } + } + + outMsg.SerializeToArcadiaStream(&Cout); +} + +void ListModules(const TString& dir) { + TVector<TString> udfPaths; + NMiniKQL::FindUdfsInDir(dir, &udfPaths); + auto funcRegistry = CreateFunctionRegistry(&NYql::NBacktrace::KikimrBackTrace, IBuiltinFunctionRegistry::TPtr(), false, udfPaths, + NUdf::IRegistrator::TFlags::TypesOnly); + + for (auto& m : funcRegistry->GetAllModuleNames()) { + auto path = *funcRegistry->FindUdfPath(m); + Cout << m << '\t' << path << Endl; + } +} + +// NOLINTBEGIN(readability-identifier-naming) +#ifdef _linux_ +struct my_siginfo_t + { + int si_signo; /* Signal number. */ +#if __SI_ERRNO_THEN_CODE + int si_errno; /* If non-zero, an errno value associated with + this signal, as defined in <errno.h>. */ + int si_code; /* Signal code. */ +#else + int si_code; + int si_errno; +#endif +#if __WORDSIZE == 64 + int __pad0; /* Explicit padding. */ +#endif + union + { + int _pad[__SI_PAD_SIZE]; + struct + { + void *_call_addr; /* Calling user insn. */ + int _syscall; /* Triggering system call number. */ + unsigned int _arch; /* AUDIT_ARCH_* of syscall. */ + } _sigsys; + + } _sifields; + }; +// NOLINTEND(readability-identifier-naming) + +void SigSysHandler(int sig, my_siginfo_t *info, void *) { + Cerr << "SigSysHandler: " << sig << ", code: " << info->si_code << ", errno: " << + info->si_errno << ", call: " << info->_sifields._sigsys._syscall << ", arch:" << info->_sifields._sigsys._arch << "\n"; + // repeat SIGSYS signal (this will kill current process) + raise(sig); +} +#endif + +int main(int argc, char **argv) { + NYql::NBacktrace::RegisterKikimrFatalActions(); + NYql::NBacktrace::EnableKikimrSymbolize(); + + try { +#ifdef _linux_ + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_flags = SA_RESETHAND | SA_SIGINFO; + typedef void (*TSigSysHandler)(int, siginfo_t *, void *); + sa.sa_sigaction = (TSigSysHandler)SigSysHandler; + sigfillset(&sa.sa_mask); + if (sigaction(SIGSYS, &sa, nullptr) == -1) { + ythrow TSystemError() << "Cannot set handler for signal " << strsignal(SIGSYS); + } +#endif + + TString path; + TString user; + TString group; + bool printAsProto = true; + + NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default(); + opts.AddLongOption('L', "list", "List UDF modules in specified directory") + .Optional() + .RequiredArgument("DIR") + .StoreResult(&path); + + opts.AddLongOption('D', "discover-path", "Discover UDFs in folder or single file") + .Optional() + .RequiredArgument("DIR") + .StoreResult(&path); + + opts.AddLongOption('P', "discover-proto", "Discover UDFs according to TResolve proto from cin") + .Optional() + .NoArgument(); + + opts.AddLongOption('A', "as-proto", "Print result in protobuf format") + .Optional() + .DefaultValue(true) + .StoreResult(&printAsProto); + + opts.AddLongOption('U', "user", "Run as user") + .Optional() + .StoreResult(&user); + + opts.AddLongOption('G', "group", "Run as group") + .Optional() + .StoreResult(&group); + + opts.AddLongOption('F', "filter-syscalls", "Filter syscalls") + .Optional() + .NoArgument(); + + opts.SetFreeArgsNum(0); + + NLastGetopt::TOptsParseResult res(&opts, argc, argv); + + SetEnv("USER", user ? user : "udf_resolver"); + if (user && !NYql::IsLinuxKernelBelow4_3()) { + // since we are going to load untrusted modules + // we have to switch into another user and drop privileges (capabilities) + NYql::BecomeUser(user, group); + NYql::TurnOffBecomeUserAbility(); + } + + NYql::SendSignalOnParentThreadExit(SIGTERM); + + if (res.Has("filter-syscalls")) { +#ifdef _linux_ + +#define ArchField offsetof(struct seccomp_data, arch) // NOLINT(readability-identifier-naming) + +// NOLINTNEXTLINE(readability-identifier-naming) +#define Allow(syscall) \ + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, SYS_##syscall, 0, 1), \ + BPF_STMT(BPF_RET+BPF_K, SECCOMP_RET_ALLOW) + + struct sock_filter filter[] = { + /* validate arch */ + BPF_STMT(BPF_LD+BPF_W+BPF_ABS, ArchField), + BPF_JUMP( BPF_JMP+BPF_JEQ+BPF_K, AUDIT_ARCH_X86_64, 1, 0), + BPF_STMT(BPF_RET+BPF_K, SECCOMP_RET_TRAP), + + /* load syscall */ + BPF_STMT(BPF_LD+BPF_W+BPF_ABS, offsetof(struct seccomp_data, nr)), + + /* list of allowed syscalls */ +#ifndef _arm64_ + Allow(access), +#endif + Allow(brk), + Allow(chdir), + Allow(clock_gettime), + Allow(clone), + Allow(close), +#ifndef _arm64_ + Allow(creat), +#endif + Allow(dup), +#ifndef _arm64_ + Allow(dup2), +#endif + Allow(dup3), + Allow(eventfd2), + Allow(exit), + Allow(exit_group), + Allow(fadvise64), + Allow(fallocate), + Allow(flock), + Allow(fstat), + Allow(fsync), + Allow(ftruncate), + Allow(futex), + Allow(get_robust_list), + Allow(getcwd), +#ifndef _arm64_ + Allow(getdents), +#endif + Allow(getdents64), + Allow(getegid), + Allow(geteuid), + Allow(getgid), + Allow(getgroups), + Allow(getpgid), +#ifndef _arm64_ + Allow(getpgrp), +#endif + Allow(getpid), + Allow(getppid), + Allow(getpriority), + Allow(getrandom), + Allow(getrlimit), + Allow(getrusage), + Allow(getsid), + Allow(gettid), + Allow(gettimeofday), + Allow(getuid), + Allow(getxattr), + Allow(ioctl), + Allow(lgetxattr), +#ifndef _arm64_ + Allow(link), +#endif + Allow(listxattr), + Allow(llistxattr), + Allow(lremovexattr), + Allow(lseek), + Allow(lsetxattr), +#ifndef _arm64_ + Allow(lstat), +#endif + Allow(madvise), +#ifndef _arm64_ + Allow(mkdir), +#endif + Allow(mkdirat), + Allow(mlock), + Allow(mlockall), + Allow(mmap), + Allow(mprotect), + Allow(munlock), + Allow(munlockall), + Allow(munmap), + Allow(nanosleep), +#ifndef _arm64_ + Allow(open), +#endif + Allow(openat), + Allow(pipe2), + Allow(prctl), + Allow(pread64), + Allow(pwrite64), + Allow(read), +#ifndef _arm64_ + Allow(readlink), +#endif + Allow(readv), + Allow(removexattr), +#ifndef _arm64_ + Allow(rename), +#endif + Allow(renameat), +#ifndef _arm64_ + Allow(rmdir), +#endif + Allow(rt_sigaction), + Allow(rt_sigpending), + Allow(rt_sigprocmask), + Allow(rt_sigqueueinfo), + Allow(rt_sigreturn), + Allow(rt_sigsuspend), + Allow(rt_sigtimedwait), + Allow(rt_tgsigqueueinfo), + Allow(sched_getaffinity), + Allow(sched_setaffinity), + Allow(set_robust_list), + Allow(setxattr), +#ifndef _arm64_ + Allow(stat), +#endif + Allow(sysinfo), + Allow(uname), +#ifndef _arm64_ + Allow(unlink), +#endif + Allow(unlinkat), + Allow(write), + Allow(writev), + + /* and if we don't match above, die */ + BPF_STMT(BPF_RET+BPF_K, SECCOMP_RET_TRAP), + }; + struct sock_fprog filterprog = { + .len = sizeof(filter)/sizeof(filter[0]), + .filter = filter + }; + + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) == -1) { + ythrow yexception() << "prctl(PR_SET_NO_NEW_PRIVS, 1, ...) failed with: " << LastSystemErrorText(); + } + if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &filterprog) == -1) { + ythrow yexception() << "prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, ...) failed with: " << LastSystemErrorText(); + } +#endif + } + + Y_UNUSED(NUdf::GetStaticSymbols()); + if (res.Has("list")) { + ListModules(path); + return 0; + } + + if (res.Has("discover-path")) { + NFs::EnsureExists(path); + TFileStat fstat(path); + if (fstat.IsDir()) { + NUdfResolver::DiscoverInDir(path, Cout, printAsProto); + } else { + NUdfResolver::DiscoverInFile(path, Cout, printAsProto); + } + return 0; + } + + if (res.Has("discover-proto")) { + NUdfResolver::Discover(Cin, Cout, printAsProto); + return 0; + } + + ResolveUDFs(); + return 0; + } catch (...) { + Cerr << CurrentExceptionMessage() << Endl; + return 1; + } +} diff --git a/ydb/library/yql/tools/udf_resolver/ya.make b/ydb/library/yql/tools/udf_resolver/ya.make new file mode 100644 index 0000000000..71a8344e19 --- /dev/null +++ b/ydb/library/yql/tools/udf_resolver/ya.make @@ -0,0 +1,33 @@ +PROGRAM() + +SRCS( + udf_resolver.cpp + discover.cpp +) + +IF (OS_LINUX) + # prevent external python extensions to lookup protobuf symbols (and maybe + # other common stuff) in main binary + EXPORTS_SCRIPT(${ARCADIA_ROOT}/ydb/library/yql/tools/exports.symlist) + + PEERDIR( + contrib/libs/libc_compat + ) +ENDIF() + +PEERDIR( + library/cpp/getopt + library/cpp/protobuf/util + ydb/library/yql/minikql + ydb/library/yql/public/udf/service/terminate_policy + ydb/library/yql/core + ydb/library/yql/providers/common/proto + ydb/library/yql/providers/common/schema/mkql + ydb/library/yql/utils/backtrace + ydb/library/yql/utils/sys + ydb/library/yql/sql/pg_dummy +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/library/yql/tools/ya.make b/ydb/library/yql/tools/ya.make index 5ff8e2b273..4631e3b04e 100644 --- a/ydb/library/yql/tools/ya.make +++ b/ydb/library/yql/tools/ya.make @@ -5,5 +5,8 @@ RECURSE( mrjob sql2yql sql_formatter + udf_dep_stub + udf_probe + udf_resolver yqlrun ) |