aboutsummaryrefslogtreecommitdiffstats
path: root/yt/yql
diff options
context:
space:
mode:
authormax42 <max42@yandex-team.com>2023-07-29 00:02:16 +0300
committermax42 <max42@yandex-team.com>2023-07-29 00:02:16 +0300
commit73b89de71748a21e102d27b9f3ed1bf658766cb5 (patch)
tree188bbd2d622fa91cdcbb1b6d6d77fbc84a0646f5 /yt/yql
parent528e321bcc2a2b67b53aeba58c3bd88305a141ee (diff)
downloadydb-73b89de71748a21e102d27b9f3ed1bf658766cb5.tar.gz
YT-19210: expose YQL shared library for YT.
After this, a new target libyqlplugin.so appears. in open-source cmake build. Diff in open-source YDB repo looks like the following: https://paste.yandex-team.ru/f302bdb4-7ef2-4362-91c7-6ca45f329264
Diffstat (limited to 'yt/yql')
-rw-r--r--yt/yql/CMakeLists.txt9
-rw-r--r--yt/yql/plugin/CMakeLists.darwin-x86_64.txt19
-rw-r--r--yt/yql/plugin/CMakeLists.linux-aarch64.txt20
-rw-r--r--yt/yql/plugin/CMakeLists.linux-x86_64.txt20
-rw-r--r--yt/yql/plugin/CMakeLists.txt17
-rw-r--r--yt/yql/plugin/CMakeLists.windows-x86_64.txt19
-rw-r--r--yt/yql/plugin/bridge/interface.h74
-rw-r--r--yt/yql/plugin/bridge/plugin.cpp120
-rw-r--r--yt/yql/plugin/bridge/plugin.h13
-rw-r--r--yt/yql/plugin/bridge/ya.make11
-rw-r--r--yt/yql/plugin/dynamic/CMakeLists.darwin-x86_64.txt31
-rw-r--r--yt/yql/plugin/dynamic/CMakeLists.linux-aarch64.txt35
-rw-r--r--yt/yql/plugin/dynamic/CMakeLists.linux-x86_64.txt35
-rw-r--r--yt/yql/plugin/dynamic/CMakeLists.txt17
-rw-r--r--yt/yql/plugin/dynamic/CMakeLists.windows-x86_64.txt22
-rw-r--r--yt/yql/plugin/dynamic/dylib.exports12
-rw-r--r--yt/yql/plugin/dynamic/impl.cpp93
-rw-r--r--yt/yql/plugin/dynamic/ya.make15
-rw-r--r--yt/yql/plugin/native/CMakeLists.darwin-x86_64.txt92
-rw-r--r--yt/yql/plugin/native/CMakeLists.linux-aarch64.txt94
-rw-r--r--yt/yql/plugin/native/CMakeLists.linux-x86_64.txt94
-rw-r--r--yt/yql/plugin/native/CMakeLists.txt17
-rw-r--r--yt/yql/plugin/native/CMakeLists.windows-x86_64.txt92
-rw-r--r--yt/yql/plugin/native/error_helpers.cpp121
-rw-r--r--yt/yql/plugin/native/error_helpers.h17
-rw-r--r--yt/yql/plugin/native/plugin.cpp293
-rw-r--r--yt/yql/plugin/native/plugin.h14
-rw-r--r--yt/yql/plugin/native/ya.make44
-rw-r--r--yt/yql/plugin/plugin.cpp18
-rw-r--r--yt/yql/plugin/plugin.h65
-rw-r--r--yt/yql/plugin/ya.make19
31 files changed, 1562 insertions, 0 deletions
diff --git a/yt/yql/CMakeLists.txt b/yt/yql/CMakeLists.txt
new file mode 100644
index 0000000000..8180a508fa
--- /dev/null
+++ b/yt/yql/CMakeLists.txt
@@ -0,0 +1,9 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(plugin)
diff --git a/yt/yql/plugin/CMakeLists.darwin-x86_64.txt b/yt/yql/plugin/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..e4e42a5389
--- /dev/null
+++ b/yt/yql/plugin/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,19 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(dynamic)
+add_subdirectory(native)
+
+add_library(yt-yql-plugin)
+target_link_libraries(yt-yql-plugin PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+)
+target_sources(yt-yql-plugin PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/plugin.cpp
+)
diff --git a/yt/yql/plugin/CMakeLists.linux-aarch64.txt b/yt/yql/plugin/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..3fb7f34a94
--- /dev/null
+++ b/yt/yql/plugin/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,20 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(dynamic)
+add_subdirectory(native)
+
+add_library(yt-yql-plugin)
+target_link_libraries(yt-yql-plugin PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+)
+target_sources(yt-yql-plugin PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/plugin.cpp
+)
diff --git a/yt/yql/plugin/CMakeLists.linux-x86_64.txt b/yt/yql/plugin/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..3fb7f34a94
--- /dev/null
+++ b/yt/yql/plugin/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,20 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(dynamic)
+add_subdirectory(native)
+
+add_library(yt-yql-plugin)
+target_link_libraries(yt-yql-plugin PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+)
+target_sources(yt-yql-plugin PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/plugin.cpp
+)
diff --git a/yt/yql/plugin/CMakeLists.txt b/yt/yql/plugin/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/yt/yql/plugin/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/yt/yql/plugin/CMakeLists.windows-x86_64.txt b/yt/yql/plugin/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..e4e42a5389
--- /dev/null
+++ b/yt/yql/plugin/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,19 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(dynamic)
+add_subdirectory(native)
+
+add_library(yt-yql-plugin)
+target_link_libraries(yt-yql-plugin PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+)
+target_sources(yt-yql-plugin PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/plugin.cpp
+)
diff --git a/yt/yql/plugin/bridge/interface.h b/yt/yql/plugin/bridge/interface.h
new file mode 100644
index 0000000000..062adb7ea5
--- /dev/null
+++ b/yt/yql/plugin/bridge/interface.h
@@ -0,0 +1,74 @@
+#pragma once
+
+#include <unistd.h>
+
+////////////////////////////////////////////////////////////////////////////////
+
+// This is a plain C version of an interface described in yt/yql/plugin/plugin.h.
+// All strings without separate length field are assumed to be null-terminated.
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TBridgeYqlPluginOptions
+{
+ const char* MRJobBinary;
+ const char* UdfDirectory;
+
+ struct TBridgeCluster
+ {
+ const char* Cluster;
+ const char* Proxy;
+ };
+ ssize_t ClusterCount;
+ TBridgeCluster* Clusters;
+ const char* DefaultCluster;
+
+ const char* OperationAttributes;
+
+ const char* YTTokenPath;
+
+ // TODO(max42): passing C++ objects across shared libraries is incredibly
+ // fragile. This is a temporary mean until we come up with something more
+ // convenient; get rid of this ASAP.
+ using TLogBackendHolder = void;
+ TLogBackendHolder* LogBackend;
+};
+
+// Opaque type representing a YQL plugin.
+using TBridgeYqlPlugin = void;
+
+using TFuncBridgeCreateYqlPlugin = TBridgeYqlPlugin*(const TBridgeYqlPluginOptions* options);
+using TFuncBridgeFreeYqlPlugin = void(TBridgeYqlPlugin* plugin);
+
+////////////////////////////////////////////////////////////////////////////////
+
+// TODO(max42): consider making structure an opaque type with accessors a-la
+// const char* BridgeGetYsonResult(const TBridgeQueryResult*). This would remove the need
+// to manually free string data.
+struct TBridgeQueryResult
+{
+ const char* YsonResult = nullptr;
+ ssize_t YsonResultLength = 0;
+ const char* Plan = nullptr;
+ ssize_t PlanLength = 0;
+ const char* Statistics = nullptr;
+ ssize_t StatisticsLength = 0;
+ const char* TaskInfo = nullptr;
+ ssize_t TaskInfoLength = 0;
+
+ const char* YsonError = nullptr;
+ ssize_t YsonErrorLength = 0;
+};
+
+using TFuncBridgeFreeQueryResult = void(TBridgeQueryResult* result);
+using TFuncBridgeRun = TBridgeQueryResult*(TBridgeYqlPlugin* plugin, const char* impersonationUser, const char* queryText, const char* settings);
+
+////////////////////////////////////////////////////////////////////////////////
+
+#define FOR_EACH_BRIDGE_INTERFACE_FUNCTION(XX) \
+ XX(BridgeCreateYqlPlugin) \
+ XX(BridgeFreeYqlPlugin) \
+ XX(BridgeFreeQueryResult) \
+ XX(BridgeRun)
+
+////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yql/plugin/bridge/plugin.cpp b/yt/yql/plugin/bridge/plugin.cpp
new file mode 100644
index 0000000000..52b9f92a01
--- /dev/null
+++ b/yt/yql/plugin/bridge/plugin.cpp
@@ -0,0 +1,120 @@
+#include "plugin.h"
+
+#include "interface.h"
+
+#include <yt/yql/plugin/plugin.h>
+#include <util/system/dynlib.h>
+
+#include <vector>
+#include <optional>
+
+namespace NYT::NYqlPlugin {
+namespace NBridge {
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TDynamicYqlPlugin
+{
+public:
+ TDynamicYqlPlugin(std::optional<TString> yqlPluginSharedLibrary)
+ {
+ const TString DefaultYqlPluginLibraryName = "./libyqlplugin.so";
+ auto sharedLibraryPath = yqlPluginSharedLibrary.value_or(DefaultYqlPluginLibraryName);
+ Library_.Open(sharedLibraryPath.data());
+ #define XX(function) function = reinterpret_cast<TFunc ## function*>(Library_.Sym(#function));
+ FOR_EACH_BRIDGE_INTERFACE_FUNCTION(XX);
+ #undef XX
+ }
+
+protected:
+ #define XX(function) TFunc ## function* function;
+ FOR_EACH_BRIDGE_INTERFACE_FUNCTION(XX)
+ #undef XX
+
+ TDynamicLibrary Library_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TYqlPlugin
+ : public TDynamicYqlPlugin
+ , public IYqlPlugin
+{
+public:
+ explicit TYqlPlugin(TYqlPluginOptions& options)
+ : TDynamicYqlPlugin(options.YqlPluginSharedLibrary)
+ {
+ std::vector<TBridgeYqlPluginOptions::TBridgeCluster> bridgeClusters;
+ for (const auto& [cluster, proxy]: options.Clusters) {
+ bridgeClusters.push_back({
+ .Cluster = cluster.data(),
+ .Proxy = proxy.data(),
+ });
+ }
+
+ const char* operationAttributes = options.OperationAttributes
+ ? options.OperationAttributes.ToString().data()
+ : nullptr;
+
+ const char* defaultCluster = options.DefaultCluster
+ ? options.DefaultCluster->data()
+ : nullptr;
+
+ TBridgeYqlPluginOptions bridgeOptions {
+ .MRJobBinary = options.MRJobBinary.data(),
+ .UdfDirectory = options.UdfDirectory.data(),
+ .ClusterCount = static_cast<int>(bridgeClusters.size()),
+ .Clusters = bridgeClusters.data(),
+ .DefaultCluster = defaultCluster,
+ .OperationAttributes = operationAttributes,
+ .YTTokenPath = options.YTTokenPath.data(),
+ .LogBackend = &options.LogBackend,
+ };
+
+ BridgePlugin_ = BridgeCreateYqlPlugin(&bridgeOptions);
+ }
+
+ TQueryResult Run(TString impersonationUser, TString queryText, NYson::TYsonString settings) noexcept override
+ {
+ const char* settingsData = settings ? settings.ToString().data() : nullptr;
+ auto* bridgeQueryResult = BridgeRun(BridgePlugin_, impersonationUser.data(), queryText.data(), settingsData);
+ auto toString = [] (const char* str, size_t strLength) -> std::optional<TString> {
+ if (!str) {
+ return std::nullopt;
+ }
+ return TString(str, strLength);
+ };
+ TQueryResult queryResult = {
+ .YsonResult = toString(bridgeQueryResult->YsonResult, bridgeQueryResult->YsonResultLength),
+ .Plan = toString(bridgeQueryResult->Plan, bridgeQueryResult->PlanLength),
+ .Statistics = toString(bridgeQueryResult->Statistics, bridgeQueryResult->StatisticsLength),
+ .TaskInfo = toString(bridgeQueryResult->TaskInfo, bridgeQueryResult->TaskInfoLength),
+ .YsonError = toString(bridgeQueryResult->YsonError, bridgeQueryResult->YsonErrorLength),
+ };
+ BridgeFreeQueryResult(bridgeQueryResult);
+ return queryResult;
+ }
+
+ ~TYqlPlugin() override
+ {
+ BridgeFreeYqlPlugin(BridgePlugin_);
+ }
+
+private:
+ TBridgeYqlPlugin* BridgePlugin_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NBridge
+
+////////////////////////////////////////////////////////////////////////////////
+
+std::unique_ptr<IYqlPlugin> CreateYqlPlugin(TYqlPluginOptions& options) noexcept
+{
+ return std::make_unique<NBridge::TYqlPlugin>(options);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NYqlPlugin::NBridge
diff --git a/yt/yql/plugin/bridge/plugin.h b/yt/yql/plugin/bridge/plugin.h
new file mode 100644
index 0000000000..a80528e700
--- /dev/null
+++ b/yt/yql/plugin/bridge/plugin.h
@@ -0,0 +1,13 @@
+#pragma once
+
+#include <yt/yql/plugin/plugin.h>
+
+namespace NYT::NYqlPlugin {
+
+////////////////////////////////////////////////////////////////////////////////
+
+std::unique_ptr<IYqlPlugin> CreateYqlPlugin(TYqlPluginOptions& options) noexcept;
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NYqlPlugin
diff --git a/yt/yql/plugin/bridge/ya.make b/yt/yql/plugin/bridge/ya.make
new file mode 100644
index 0000000000..93425c5284
--- /dev/null
+++ b/yt/yql/plugin/bridge/ya.make
@@ -0,0 +1,11 @@
+LIBRARY()
+
+SRCS(
+ GLOBAL plugin.cpp
+)
+
+PEERDIR(
+ yt/yql/plugin
+)
+
+END()
diff --git a/yt/yql/plugin/dynamic/CMakeLists.darwin-x86_64.txt b/yt/yql/plugin/dynamic/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..bb76915330
--- /dev/null
+++ b/yt/yql/plugin/dynamic/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,31 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_shared_library(yqlplugin)
+target_link_libraries(yqlplugin PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yql-plugin-native
+)
+target_link_options(yqlplugin PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -undefined
+ dynamic_lookup
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(yqlplugin PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/dynamic/impl.cpp
+)
+use_export_script(yqlplugin
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/dynamic/dylib.exports
+)
+vcs_info(yqlplugin)
diff --git a/yt/yql/plugin/dynamic/CMakeLists.linux-aarch64.txt b/yt/yql/plugin/dynamic/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..63edf774f2
--- /dev/null
+++ b/yt/yql/plugin/dynamic/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,35 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_shared_library(yqlplugin)
+target_link_libraries(yqlplugin PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yql-plugin-native
+)
+target_link_options(yqlplugin PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -Wl,-z,notext
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+ -lutil
+)
+target_sources(yqlplugin PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/dynamic/impl.cpp
+)
+use_export_script(yqlplugin
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/dynamic/dylib.exports
+)
+vcs_info(yqlplugin)
diff --git a/yt/yql/plugin/dynamic/CMakeLists.linux-x86_64.txt b/yt/yql/plugin/dynamic/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..63edf774f2
--- /dev/null
+++ b/yt/yql/plugin/dynamic/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,35 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_shared_library(yqlplugin)
+target_link_libraries(yqlplugin PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yql-plugin-native
+)
+target_link_options(yqlplugin PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -Wl,-z,notext
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+ -lutil
+)
+target_sources(yqlplugin PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/dynamic/impl.cpp
+)
+use_export_script(yqlplugin
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/dynamic/dylib.exports
+)
+vcs_info(yqlplugin)
diff --git a/yt/yql/plugin/dynamic/CMakeLists.txt b/yt/yql/plugin/dynamic/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/yt/yql/plugin/dynamic/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/yt/yql/plugin/dynamic/CMakeLists.windows-x86_64.txt b/yt/yql/plugin/dynamic/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..2814417000
--- /dev/null
+++ b/yt/yql/plugin/dynamic/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,22 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_shared_library(yqlplugin)
+target_link_libraries(yqlplugin PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yql-plugin-native
+)
+target_sources(yqlplugin PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/dynamic/impl.cpp
+)
+use_export_script(yqlplugin
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/dynamic/dylib.exports
+)
+vcs_info(yqlplugin)
diff --git a/yt/yql/plugin/dynamic/dylib.exports b/yt/yql/plugin/dynamic/dylib.exports
new file mode 100644
index 0000000000..ec1c95cb11
--- /dev/null
+++ b/yt/yql/plugin/dynamic/dylib.exports
@@ -0,0 +1,12 @@
+# YT <-> YQL bridge.
+BridgeCreateYqlPlugin
+BridgeFreeYqlPlugin
+BridgeFreeQueryResult
+BridgeRun
+
+# YQL <-> YQL UDFs interface.
+UdfAllocateWithSize
+UdfFreeWithSize
+UdfRegisterObject
+UdfTerminate
+UdfUnregisterObject
diff --git a/yt/yql/plugin/dynamic/impl.cpp b/yt/yql/plugin/dynamic/impl.cpp
new file mode 100644
index 0000000000..e39897c97e
--- /dev/null
+++ b/yt/yql/plugin/dynamic/impl.cpp
@@ -0,0 +1,93 @@
+#include <yt/yql/plugin/bridge/interface.h>
+#include <yt/yql/plugin/native/plugin.h>
+
+#include <type_traits>
+
+using namespace NYT::NYqlPlugin;
+using namespace NYT::NYson;
+
+extern "C" {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TBridgeYqlPlugin* BridgeCreateYqlPlugin(const TBridgeYqlPluginOptions* bridgeOptions)
+{
+ THashMap<TString, TString> clusters;
+ for (auto clusterIndex = 0; clusterIndex < bridgeOptions->ClusterCount; ++clusterIndex) {
+ const auto& Cluster = bridgeOptions->Clusters[clusterIndex];
+ clusters[Cluster.Cluster] = Cluster.Proxy;
+ }
+
+ TYsonString operationAttributes = bridgeOptions->OperationAttributes
+ ? TYsonString(TString(bridgeOptions->OperationAttributes))
+ : TYsonString{};
+
+ TYqlPluginOptions options{
+ .MRJobBinary = TString(bridgeOptions->MRJobBinary),
+ .UdfDirectory = TString(bridgeOptions->UdfDirectory),
+ .Clusters = std::move(clusters),
+ .DefaultCluster = std::optional<TString>(bridgeOptions->DefaultCluster),
+ .OperationAttributes = operationAttributes,
+ .YTTokenPath = TString(bridgeOptions->YTTokenPath),
+ .LogBackend = std::move(*reinterpret_cast<THolder<TLogBackend>*>(bridgeOptions->LogBackend)),
+ };
+ auto nativePlugin = CreateYqlPlugin(options);
+ return nativePlugin.release();
+}
+
+void BridgeFreeYqlPlugin(TBridgeYqlPlugin* plugin)
+{
+ auto* nativePlugin = reinterpret_cast<IYqlPlugin*>(plugin);
+ delete nativePlugin;
+}
+
+void BridgeFreeQueryResult(TBridgeQueryResult* result)
+{
+ delete result->TaskInfo;
+ delete result->Statistics;
+ delete result->Plan;
+ delete result->YsonResult;
+ delete result->YsonError;
+ delete result;
+}
+
+TBridgeQueryResult* BridgeRun(TBridgeYqlPlugin* plugin, const char* impersonationUser, const char* queryText, const char* settings)
+{
+ static const TYsonString EmptyMap = TYsonString(TString("{}"));
+
+ auto* nativePlugin = reinterpret_cast<IYqlPlugin*>(plugin);
+ auto* bridgeResult = new TBridgeQueryResult;
+
+ auto fillString = [] (const char*& str, ssize_t& strLength, const std::optional<TString>& original) {
+ if (!original) {
+ str = nullptr;
+ strLength = 0;
+ return;
+ }
+ char* copy = new char[original->size() + 1];
+ memcpy(copy, original->data(), original->size() + 1);
+ str = copy;
+ strLength = original->size();
+ };
+
+ auto result = nativePlugin->Run(TString(impersonationUser), TString(queryText), settings ? TYsonString(TString(settings)) : EmptyMap);
+ fillString(bridgeResult->YsonResult, bridgeResult->YsonResultLength, result.YsonResult);
+ fillString(bridgeResult->Plan, bridgeResult->PlanLength, result.Plan);
+ fillString(bridgeResult->Statistics, bridgeResult->StatisticsLength, result.Statistics);
+ fillString(bridgeResult->TaskInfo, bridgeResult->TaskInfoLength, result.TaskInfo);
+ fillString(bridgeResult->YsonError, bridgeResult->YsonErrorLength, result.YsonError);
+
+ return bridgeResult;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+// Validate that the all functions from the bridge interface are implemented with proper signatures.
+
+#define XX(function) static_assert(std::is_same_v<decltype(&(function)), TFunc ## function*>);
+FOR_EACH_BRIDGE_INTERFACE_FUNCTION(XX)
+#undef XX
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // extern "C"
diff --git a/yt/yql/plugin/dynamic/ya.make b/yt/yql/plugin/dynamic/ya.make
new file mode 100644
index 0000000000..c91996acec
--- /dev/null
+++ b/yt/yql/plugin/dynamic/ya.make
@@ -0,0 +1,15 @@
+DLL(yqlplugin 1 0)
+
+INCLUDE(${ARCADIA_ROOT}/yt/opensource.inc)
+
+EXPORTS_SCRIPT(dylib.exports)
+
+SRCS(
+ impl.cpp
+)
+
+PEERDIR(
+ yt/yql/plugin/native
+)
+
+END()
diff --git a/yt/yql/plugin/native/CMakeLists.darwin-x86_64.txt b/yt/yql/plugin/native/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..753bac36c8
--- /dev/null
+++ b/yt/yql/plugin/native/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,92 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yql-plugin-native)
+target_compile_options(yql-plugin-native PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(yql-plugin-native PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ contrib-libs-protobuf
+ library-cpp-resource
+ library-cpp-yson
+ cpp-yson-node
+ cpp-mapreduce-client
+ cpp-mapreduce-common
+ library-yql-ast
+ yql-sql-pg
+ yql-parser-pg_wrapper
+ yql-core-facade
+ yql-core-file_storage
+ core-file_storage-proto
+ core-file_storage-http_download
+ core-services-mounts
+ yql-core-user_data
+ library-yql-minikql
+ library-yql-protos
+ udf-service-exception_policy
+ yql-utils-backtrace
+ yql-utils-log
+ providers-common-proto
+ providers-common-udf_resolve
+ providers-solomon-gateway
+ providers-solomon-provider
+ yql-core-url_preprocessing
+ yt-gateway-native
+ yt-lib-log
+ yt-lib-yt_download
+ providers-yt-provider
+ yt-yql-plugin
+)
+target_sources(yql-plugin-native PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/native/error_helpers.cpp
+)
+
+add_global_library_for(yql-plugin-native.global yql-plugin-native)
+target_compile_options(yql-plugin-native.global PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(yql-plugin-native.global PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ contrib-libs-protobuf
+ library-cpp-resource
+ library-cpp-yson
+ cpp-yson-node
+ cpp-mapreduce-client
+ cpp-mapreduce-common
+ library-yql-ast
+ yql-sql-pg
+ yql-parser-pg_wrapper
+ yql-core-facade
+ yql-core-file_storage
+ core-file_storage-proto
+ core-file_storage-http_download
+ core-services-mounts
+ yql-core-user_data
+ library-yql-minikql
+ library-yql-protos
+ udf-service-exception_policy
+ yql-utils-backtrace
+ yql-utils-log
+ providers-common-proto
+ providers-common-udf_resolve
+ providers-solomon-gateway
+ providers-solomon-provider
+ yql-core-url_preprocessing
+ yt-gateway-native
+ yt-lib-log
+ yt-lib-yt_download
+ providers-yt-provider
+ yt-yql-plugin
+)
+target_sources(yql-plugin-native.global PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/native/plugin.cpp
+)
diff --git a/yt/yql/plugin/native/CMakeLists.linux-aarch64.txt b/yt/yql/plugin/native/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..348d9ca192
--- /dev/null
+++ b/yt/yql/plugin/native/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,94 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yql-plugin-native)
+target_compile_options(yql-plugin-native PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(yql-plugin-native PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ contrib-libs-protobuf
+ library-cpp-resource
+ library-cpp-yson
+ cpp-yson-node
+ cpp-mapreduce-client
+ cpp-mapreduce-common
+ library-yql-ast
+ yql-sql-pg
+ yql-parser-pg_wrapper
+ yql-core-facade
+ yql-core-file_storage
+ core-file_storage-proto
+ core-file_storage-http_download
+ core-services-mounts
+ yql-core-user_data
+ library-yql-minikql
+ library-yql-protos
+ udf-service-exception_policy
+ yql-utils-backtrace
+ yql-utils-log
+ providers-common-proto
+ providers-common-udf_resolve
+ providers-solomon-gateway
+ providers-solomon-provider
+ yql-core-url_preprocessing
+ yt-gateway-native
+ yt-lib-log
+ yt-lib-yt_download
+ providers-yt-provider
+ yt-yql-plugin
+)
+target_sources(yql-plugin-native PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/native/error_helpers.cpp
+)
+
+add_global_library_for(yql-plugin-native.global yql-plugin-native)
+target_compile_options(yql-plugin-native.global PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(yql-plugin-native.global PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ contrib-libs-protobuf
+ library-cpp-resource
+ library-cpp-yson
+ cpp-yson-node
+ cpp-mapreduce-client
+ cpp-mapreduce-common
+ library-yql-ast
+ yql-sql-pg
+ yql-parser-pg_wrapper
+ yql-core-facade
+ yql-core-file_storage
+ core-file_storage-proto
+ core-file_storage-http_download
+ core-services-mounts
+ yql-core-user_data
+ library-yql-minikql
+ library-yql-protos
+ udf-service-exception_policy
+ yql-utils-backtrace
+ yql-utils-log
+ providers-common-proto
+ providers-common-udf_resolve
+ providers-solomon-gateway
+ providers-solomon-provider
+ yql-core-url_preprocessing
+ yt-gateway-native
+ yt-lib-log
+ yt-lib-yt_download
+ providers-yt-provider
+ yt-yql-plugin
+)
+target_sources(yql-plugin-native.global PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/native/plugin.cpp
+)
diff --git a/yt/yql/plugin/native/CMakeLists.linux-x86_64.txt b/yt/yql/plugin/native/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..348d9ca192
--- /dev/null
+++ b/yt/yql/plugin/native/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,94 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yql-plugin-native)
+target_compile_options(yql-plugin-native PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(yql-plugin-native PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ contrib-libs-protobuf
+ library-cpp-resource
+ library-cpp-yson
+ cpp-yson-node
+ cpp-mapreduce-client
+ cpp-mapreduce-common
+ library-yql-ast
+ yql-sql-pg
+ yql-parser-pg_wrapper
+ yql-core-facade
+ yql-core-file_storage
+ core-file_storage-proto
+ core-file_storage-http_download
+ core-services-mounts
+ yql-core-user_data
+ library-yql-minikql
+ library-yql-protos
+ udf-service-exception_policy
+ yql-utils-backtrace
+ yql-utils-log
+ providers-common-proto
+ providers-common-udf_resolve
+ providers-solomon-gateway
+ providers-solomon-provider
+ yql-core-url_preprocessing
+ yt-gateway-native
+ yt-lib-log
+ yt-lib-yt_download
+ providers-yt-provider
+ yt-yql-plugin
+)
+target_sources(yql-plugin-native PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/native/error_helpers.cpp
+)
+
+add_global_library_for(yql-plugin-native.global yql-plugin-native)
+target_compile_options(yql-plugin-native.global PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(yql-plugin-native.global PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ contrib-libs-protobuf
+ library-cpp-resource
+ library-cpp-yson
+ cpp-yson-node
+ cpp-mapreduce-client
+ cpp-mapreduce-common
+ library-yql-ast
+ yql-sql-pg
+ yql-parser-pg_wrapper
+ yql-core-facade
+ yql-core-file_storage
+ core-file_storage-proto
+ core-file_storage-http_download
+ core-services-mounts
+ yql-core-user_data
+ library-yql-minikql
+ library-yql-protos
+ udf-service-exception_policy
+ yql-utils-backtrace
+ yql-utils-log
+ providers-common-proto
+ providers-common-udf_resolve
+ providers-solomon-gateway
+ providers-solomon-provider
+ yql-core-url_preprocessing
+ yt-gateway-native
+ yt-lib-log
+ yt-lib-yt_download
+ providers-yt-provider
+ yt-yql-plugin
+)
+target_sources(yql-plugin-native.global PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/native/plugin.cpp
+)
diff --git a/yt/yql/plugin/native/CMakeLists.txt b/yt/yql/plugin/native/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/yt/yql/plugin/native/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/yt/yql/plugin/native/CMakeLists.windows-x86_64.txt b/yt/yql/plugin/native/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..753bac36c8
--- /dev/null
+++ b/yt/yql/plugin/native/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,92 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(yql-plugin-native)
+target_compile_options(yql-plugin-native PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(yql-plugin-native PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ contrib-libs-protobuf
+ library-cpp-resource
+ library-cpp-yson
+ cpp-yson-node
+ cpp-mapreduce-client
+ cpp-mapreduce-common
+ library-yql-ast
+ yql-sql-pg
+ yql-parser-pg_wrapper
+ yql-core-facade
+ yql-core-file_storage
+ core-file_storage-proto
+ core-file_storage-http_download
+ core-services-mounts
+ yql-core-user_data
+ library-yql-minikql
+ library-yql-protos
+ udf-service-exception_policy
+ yql-utils-backtrace
+ yql-utils-log
+ providers-common-proto
+ providers-common-udf_resolve
+ providers-solomon-gateway
+ providers-solomon-provider
+ yql-core-url_preprocessing
+ yt-gateway-native
+ yt-lib-log
+ yt-lib-yt_download
+ providers-yt-provider
+ yt-yql-plugin
+)
+target_sources(yql-plugin-native PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/native/error_helpers.cpp
+)
+
+add_global_library_for(yql-plugin-native.global yql-plugin-native)
+target_compile_options(yql-plugin-native.global PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(yql-plugin-native.global PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ contrib-libs-protobuf
+ library-cpp-resource
+ library-cpp-yson
+ cpp-yson-node
+ cpp-mapreduce-client
+ cpp-mapreduce-common
+ library-yql-ast
+ yql-sql-pg
+ yql-parser-pg_wrapper
+ yql-core-facade
+ yql-core-file_storage
+ core-file_storage-proto
+ core-file_storage-http_download
+ core-services-mounts
+ yql-core-user_data
+ library-yql-minikql
+ library-yql-protos
+ udf-service-exception_policy
+ yql-utils-backtrace
+ yql-utils-log
+ providers-common-proto
+ providers-common-udf_resolve
+ providers-solomon-gateway
+ providers-solomon-provider
+ yql-core-url_preprocessing
+ yt-gateway-native
+ yt-lib-log
+ yt-lib-yt_download
+ providers-yt-provider
+ yt-yql-plugin
+)
+target_sources(yql-plugin-native.global PRIVATE
+ ${CMAKE_SOURCE_DIR}/yt/yql/plugin/native/plugin.cpp
+)
diff --git a/yt/yql/plugin/native/error_helpers.cpp b/yt/yql/plugin/native/error_helpers.cpp
new file mode 100644
index 0000000000..23b9fa5cba
--- /dev/null
+++ b/yt/yql/plugin/native/error_helpers.cpp
@@ -0,0 +1,121 @@
+#include "error_helpers.h"
+
+#include <library/cpp/yson/writer.h>
+
+#include <ydb/library/yql/public/issue/yql_issue_id.h>
+
+#include <ydb/library/yql/core/issue/protos/issue_id.pb.h>
+
+namespace NYT::NYqlPlugin {
+
+////////////////////////////////////////////////////////////////////////////////
+
+const int IssueToErrorCodesShift = 30000;
+
+////////////////////////////////////////////////////////////////////////////////
+
+TString ExceptionToYtErrorYson(const std::exception& exception)
+{
+ TStringStream yson;
+ ::NYson::TYsonWriter writer(&yson);
+
+ writer.OnBeginMap();
+ writer.OnKeyedItem("code");
+ writer.OnInt64Scalar(1); // Generic error
+ writer.OnKeyedItem("message");
+ writer.OnStringScalar(exception.what());
+ writer.OnKeyedItem("attributes");
+ writer.OnBeginMap();
+ writer.OnEndMap();
+
+ writer.OnEndMap();
+
+ return yson.Str();
+}
+
+TString IssuesToYtErrorYson(const NYql::TIssues& issues)
+{
+ TStringStream yson;
+ ::NYson::TYsonWriter writer(&yson);
+
+ auto serializePosition = [&] (const NYql::TPosition& position) {
+ writer.OnBeginMap();
+ {
+ writer.OnKeyedItem("column");
+ writer.OnInt64Scalar(position.Column);
+
+ writer.OnKeyedItem("row");
+ writer.OnInt64Scalar(position.Row);
+
+ if (!position.File.empty()) {
+ writer.OnKeyedItem("file");
+ writer.OnStringScalar(position.File);
+ }
+ }
+ writer.OnEndMap();
+ };
+
+ auto fn = [&] (const NYql::TIssue& issue, ui16 /*level*/) {
+ writer.OnListItem();
+ writer.OnBeginMap();
+ {
+ writer.OnKeyedItem("code");
+
+ NYql::TIssueCode code = IssueToErrorCodesShift + issue.GetCode();
+ writer.OnInt64Scalar(code);
+
+ writer.OnKeyedItem("message");
+ writer.OnStringScalar(issue.GetMessage());
+
+ writer.OnKeyedItem("attributes");
+ writer.OnBeginMap();
+ {
+ if (issue.Range().Position) {
+ writer.OnKeyedItem("start_position");
+ serializePosition(issue.Range().Position);
+ }
+
+ if (issue.Range().EndPosition) {
+ writer.OnKeyedItem("end_position");
+ serializePosition(issue.Range().EndPosition);
+ }
+
+ writer.OnKeyedItem("yql_status");
+ writer.OnStringScalar(NYql::IssueCodeToString<NYql::TIssuesIds>(issue.GetCode()));
+
+ writer.OnKeyedItem("severity");
+ writer.OnStringScalar(SeverityToString(issue.GetSeverity()));
+ }
+ writer.OnEndMap();
+
+ writer.OnKeyedItem("inner_errors");
+ writer.OnBeginList();
+ }
+ };
+
+ auto afterChildrenFn = [&] (const NYql::TIssue& /*issue*/, ui16 /*level*/) {
+ writer.OnEndList();
+ writer.OnEndMap();
+ };
+
+ writer.OnBeginMap();
+ writer.OnKeyedItem("message");
+ writer.OnStringScalar("There are some issues");
+ writer.OnKeyedItem("code");
+ writer.OnInt64Scalar(1);
+ writer.OnKeyedItem("inner_errors");
+ writer.OnBeginList();
+ for (const auto& issue : issues) {
+ WalkThroughIssues(issue, /*leafOnly*/ false, fn, afterChildrenFn);
+ }
+ writer.OnEndList();
+ writer.OnKeyedItem("attributes");
+ writer.OnBeginMap();
+ writer.OnEndMap();
+ writer.OnEndMap();
+ return yson.Str();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NYqlPLugin
diff --git a/yt/yql/plugin/native/error_helpers.h b/yt/yql/plugin/native/error_helpers.h
new file mode 100644
index 0000000000..1905502aa8
--- /dev/null
+++ b/yt/yql/plugin/native/error_helpers.h
@@ -0,0 +1,17 @@
+#pragma once
+
+#include <util/generic/string.h>
+
+#include <ydb/library/yql/public/issue/yql_issue.h>
+
+namespace NYT::NYqlPlugin {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TString IssuesToYtErrorYson(const NYql::TIssues& issues);
+
+TString ExceptionToYtErrorYson(const std::exception& exception);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NYqlPlugin
diff --git a/yt/yql/plugin/native/plugin.cpp b/yt/yql/plugin/native/plugin.cpp
new file mode 100644
index 0000000000..086f63a2f9
--- /dev/null
+++ b/yt/yql/plugin/native/plugin.cpp
@@ -0,0 +1,293 @@
+#include "plugin.h"
+
+#include "error_helpers.h"
+
+#include <ydb/library/yql/providers/yt/lib/log/yt_logger.h>
+#include <ydb/library/yql/providers/yt/lib/yt_download/yt_download.h>
+#include <ydb/library/yql/providers/yt/gateway/native/yql_yt_native.h>
+#include <ydb/library/yql/providers/yt/provider/yql_yt_provider.h>
+
+#include <ydb/library/yql/core/url_preprocessing/url_preprocessing.h>
+
+#include <ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.h>
+#include "ydb/library/yql/providers/common/proto/gateways_config.pb.h"
+#include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
+
+#include <ydb/library/yql/ast/yql_expr.h>
+#include <ydb/library/yql/minikql/mkql_function_registry.h>
+#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
+#include <ydb/library/yql/core/facade/yql_facade.h>
+#include <ydb/library/yql/core/file_storage/file_storage.h>
+#include "ydb/library/yql/core/file_storage/proto/file_storage.pb.h"
+#include <ydb/library/yql/core/services/mounts/yql_mounts.h>
+#include <ydb/library/yql/utils/log/log.h>
+#include <ydb/library/yql/utils/backtrace/backtrace.h>
+
+#include <yt/cpp/mapreduce/interface/config.h>
+#include <yt/cpp/mapreduce/interface/logging/logger.h>
+
+#include <library/cpp/yson/node/node_io.h>
+
+#include <library/cpp/yson/parser.h>
+#include <library/cpp/yson/writer.h>
+
+#include <library/cpp/resource/resource.h>
+#include <library/cpp/digest/md5/md5.h>
+
+#include <util/folder/path.h>
+#include <util/stream/file.h>
+#include <util/string/builder.h>
+#include <util/system/fs.h>
+#include <util/system/user.h>
+
+namespace NYT::NYqlPlugin {
+namespace NNative {
+
+using namespace NYson;
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TYqlPlugin
+ : public IYqlPlugin
+{
+public:
+ TYqlPlugin(TYqlPluginOptions& options)
+ {
+ try {
+ NYql::NLog::InitLogger(std::move(options.LogBackend));
+
+ auto& logger = NYql::NLog::YqlLogger();
+
+ logger.SetDefaultPriority(ELogPriority::TLOG_DEBUG);
+ for (int i = 0; i < NYql::NLog::EComponentHelpers::ToInt(NYql::NLog::EComponent::MaxValue); ++i) {
+ logger.SetComponentLevel((NYql::NLog::EComponent) i, NYql::NLog::ELevel::DEBUG);
+ }
+
+ NYql::SetYtLoggerGlobalBackend(NYT::ILogger::ELevel::DEBUG);
+ if (NYT::TConfig::Get()->Prefix.empty()) {
+ NYT::TConfig::Get()->Prefix = "//";
+ }
+
+ auto yqlCoreFlags = GatewaysConfig_.GetYqlCore()
+ .GetFlags();
+
+ auto ytConfig = GatewaysConfig_.MutableYt();
+ if (!ytConfig->HasExecuteUdfLocallyIfPossible()) {
+ ytConfig->SetExecuteUdfLocallyIfPossible(true);
+ }
+
+ ytConfig->SetYtLogLevel(NYql::EYtLogLevel::YL_DEBUG);
+ ytConfig->SetMrJobBin(options.MRJobBinary);
+ ytConfig->SetMrJobBinMd5(MD5::File(options.MRJobBinary));
+
+ ytConfig->ClearMrJobUdfsDir();
+
+ for (const auto& [cluster, address]: options.Clusters) {
+ auto item = ytConfig->AddClusterMapping();
+ item->SetName(cluster);
+ item->SetCluster(address);
+ if (cluster == options.DefaultCluster) {
+ item->SetDefault(true);
+ }
+
+ Clusters_.insert({item->GetName(), TString(NYql::YtProviderName)});
+ }
+ DefaultCluster_ = options.DefaultCluster;
+
+ NYql::TFileStorageConfig fileStorageConfig;
+ fileStorageConfig.SetMaxSizeMb(1 << 14);
+ FileStorage_ = WithAsync(CreateFileStorage(fileStorageConfig, {MakeYtDownloader(fileStorageConfig)}));
+
+ FuncRegistry_ = NKikimr::NMiniKQL::CreateFunctionRegistry(
+ NKikimr::NMiniKQL::CreateBuiltinRegistry())->Clone();
+
+ const NKikimr::NMiniKQL::TUdfModuleRemappings emptyRemappings;
+
+ FuncRegistry_->SetBackTraceCallback(&NYql::NBacktrace::KikimrBackTrace);
+
+ NKikimr::NMiniKQL::TUdfModulePathsMap systemModules;
+
+ TVector<TString> udfPaths;
+ NKikimr::NMiniKQL::FindUdfsInDir(options.UdfDirectory, &udfPaths);
+ for (const auto& path: udfPaths) {
+ // Skip YQL plugin shared library itself, it is not a UDF.
+ if (path.EndsWith("libyqlplugin.so")) {
+ continue;
+ }
+ FuncRegistry_->LoadUdfs(path, emptyRemappings, 0);
+ }
+
+ for (auto& m: FuncRegistry_->GetAllModuleNames()) {
+ TMaybe<TString> path = FuncRegistry_->FindUdfPath(m);
+ if (!path) {
+ YQL_LOG(FATAL) << "Unable to detect UDF path for module " << m;
+ exit(1);
+ }
+ systemModules.emplace(m, *path);
+ }
+
+ FuncRegistry_->SetSystemModulePaths(systemModules);
+
+ NYql::TUserDataTable userDataTable = GetYqlModuleResolver(ExprContext_, ModuleResolver_, {}, Clusters_, {});
+
+ if (!userDataTable) {
+ TStringStream err;
+ ExprContext_.IssueManager
+ .GetIssues()
+ .PrintTo(err);
+ YQL_LOG(FATAL) << "Failed to compile modules:\n"
+ << err.Str();
+ exit(1);
+ }
+
+ OperationAttributes_ = options.OperationAttributes;
+
+ TVector<NYql::TDataProviderInitializer> dataProvidersInit;
+
+ NYql::TYtNativeServices ytServices;
+ ytServices.FunctionRegistry = FuncRegistry_.Get();
+ ytServices.FileStorage = FileStorage_;
+ ytServices.Config = std::make_shared<NYql::TYtGatewayConfig>(*ytConfig);
+ auto ytNativeGateway = CreateYtNativeGateway(ytServices);
+ dataProvidersInit.push_back(GetYtNativeDataProviderInitializer(ytNativeGateway));
+
+ ProgramFactory_ = std::make_unique<NYql::TProgramFactory>(
+ false, FuncRegistry_.Get(), ExprContext_.NextUniqueId, dataProvidersInit, "embedded");
+ YTTokenPath_ = options.YTTokenPath;
+ ProgramFactory_->AddUserDataTable(userDataTable);
+ ProgramFactory_->SetModules(ModuleResolver_);
+ ProgramFactory_->SetUdfResolver(NYql::NCommon::CreateSimpleUdfResolver(FuncRegistry_.Get(), FileStorage_));
+ ProgramFactory_->SetGatewaysConfig(&GatewaysConfig_);
+ ProgramFactory_->SetFileStorage(FileStorage_);
+ ProgramFactory_->SetUrlPreprocessing(MakeIntrusive<NYql::TUrlPreprocessing>(GatewaysConfig_));
+ } catch (const std::exception& ex) {
+ YQL_LOG(FATAL) << "Unexpected exception while initializing YQL plugin: " << ex.what();
+ exit(1);
+ }
+ YQL_LOG(INFO) << "YQL plugin initialized";
+ }
+
+ TQueryResult GuardedRun(TString impersonationUser, TString queryText, TYsonString settings)
+ {
+ auto credentials = MakeIntrusive<NYql::TCredentials>();
+ if (YTTokenPath_) {
+ TFsPath path(YTTokenPath_);
+ auto token = TIFStream(path).ReadAll();
+
+ credentials->AddCredential("default_yt", NYql::TCredential("yt", "", token));
+ }
+
+ credentials->AddCredential("impersonation_user_yt", NYql::TCredential("yt", "", impersonationUser));
+ ProgramFactory_->SetCredentials(credentials);
+
+ auto program = ProgramFactory_->Create("-memory-", queryText);
+ program->SetOperationAttrsYson(PatchQueryAttributes(OperationAttributes_, settings));
+
+ NSQLTranslation::TTranslationSettings sqlSettings;
+ sqlSettings.ClusterMapping = Clusters_;
+ sqlSettings.ModuleMapping = Modules_;
+ if (DefaultCluster_) {
+ sqlSettings.DefaultCluster = *DefaultCluster_;
+ }
+ sqlSettings.SyntaxVersion = 1;
+ sqlSettings.V0Behavior = NSQLTranslation::EV0Behavior::Disable;
+
+ if (!program->ParseSql(sqlSettings)) {
+ return TQueryResult{
+ .YsonError = IssuesToYtErrorYson(program->Issues()),
+ };
+ }
+
+ if (!program->Compile(GetUsername())) {
+ return TQueryResult{
+ .YsonError = IssuesToYtErrorYson(program->Issues()),
+ };
+ }
+
+ NYql::TProgram::TStatus status = NYql::TProgram::TStatus::Error;
+ status = program->Run(GetUsername(), nullptr, nullptr, nullptr);
+
+ if (status == NYql::TProgram::TStatus::Error) {
+ return TQueryResult{
+ .YsonError = IssuesToYtErrorYson(program->Issues()),
+ };
+ }
+
+ TStringStream result;
+ if (program->HasResults()) {
+ ::NYson::TYsonWriter yson(&result, EYsonFormat::Binary);
+ yson.OnBeginList();
+ for (const auto& result: program->Results()) {
+ yson.OnListItem();
+ yson.OnRaw(result);
+ }
+ yson.OnEndList();
+ }
+
+ auto maybeToOptional = [] (const TMaybe<TString>& maybeStr) -> std::optional<TString> {
+ if (!maybeStr) {
+ return std::nullopt;
+ }
+ return *maybeStr;
+ };
+
+ return {
+ .YsonResult = result.Empty() ? std::nullopt : std::make_optional(result.Str()),
+ .Plan = maybeToOptional(program->GetQueryPlan()),
+ .Statistics = maybeToOptional(program->GetStatistics()),
+ .TaskInfo = maybeToOptional(program->GetTasksInfo()),
+ };
+ }
+
+ TQueryResult Run(TString impersonationUser, TString queryText, TYsonString settings) noexcept override
+ {
+ try {
+ return GuardedRun(impersonationUser, queryText, settings);
+ } catch (const std::exception& ex) {
+ return TQueryResult{
+ .YsonError = ExceptionToYtErrorYson(ex),
+ };
+ }
+ }
+
+private:
+ NYql::TFileStoragePtr FileStorage_;
+ NYql::TExprContext ExprContext_;
+ ::TIntrusivePtr<NKikimr::NMiniKQL::IMutableFunctionRegistry> FuncRegistry_;
+ NYql::IModuleResolver::TPtr ModuleResolver_;
+ NYql::TGatewaysConfig GatewaysConfig_;
+ std::unique_ptr<NYql::TProgramFactory> ProgramFactory_;
+ TString YTTokenPath_;
+ THashMap<TString, TString> Clusters_;
+ std::optional<TString> DefaultCluster_;
+ THashMap<TString, TString> Modules_;
+ THashSet<TString> Libraries_;
+ TYsonString OperationAttributes_;
+
+ TString PatchQueryAttributes(TYsonString configAttributes, TYsonString querySettings)
+ {
+ NYT::TNode querySettingsMap = NodeFromYsonString(querySettings.ToString());
+ NYT::TNode resultAttributesMap = NodeFromYsonString(configAttributes.ToString());
+
+ for (const auto& item: querySettingsMap.AsMap()) {
+ resultAttributesMap[item.first] = item.second;
+ }
+
+ return NodeToYsonString(resultAttributesMap);
+ }
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NNative
+
+////////////////////////////////////////////////////////////////////////////////
+
+std::unique_ptr<IYqlPlugin> CreateYqlPlugin(TYqlPluginOptions& options) noexcept
+{
+ return std::make_unique<NNative::TYqlPlugin>(options);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NYqlPlugin
diff --git a/yt/yql/plugin/native/plugin.h b/yt/yql/plugin/native/plugin.h
new file mode 100644
index 0000000000..53bb7119ce
--- /dev/null
+++ b/yt/yql/plugin/native/plugin.h
@@ -0,0 +1,14 @@
+#pragma once
+
+#include <yt/yql/plugin/plugin.h>
+
+
+namespace NYT::NYqlPlugin {
+
+////////////////////////////////////////////////////////////////////////////////
+
+std::unique_ptr<IYqlPlugin> CreateYqlPlugin(TYqlPluginOptions& options) noexcept;
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NYqlPlugin
diff --git a/yt/yql/plugin/native/ya.make b/yt/yql/plugin/native/ya.make
new file mode 100644
index 0000000000..36d562a4d2
--- /dev/null
+++ b/yt/yql/plugin/native/ya.make
@@ -0,0 +1,44 @@
+LIBRARY()
+
+SRCS(
+ GLOBAL plugin.cpp
+ error_helpers.cpp
+)
+
+PEERDIR(
+ contrib/libs/protobuf
+ library/cpp/resource
+ library/cpp/yson
+ library/cpp/yson/node
+ yt/cpp/mapreduce/client
+ yt/cpp/mapreduce/common
+ ydb/library/yql/ast
+ ydb/library/yql/sql/pg
+ ydb/library/yql/parser/pg_wrapper
+ ydb/library/yql/core/facade
+ ydb/library/yql/core/file_storage
+ ydb/library/yql/core/file_storage/proto
+ ydb/library/yql/core/file_storage/http_download
+ ydb/library/yql/core/services/mounts
+ ydb/library/yql/core/user_data
+ ydb/library/yql/minikql
+ ydb/library/yql/protos
+ ydb/library/yql/public/udf/service/exception_policy
+ ydb/library/yql/utils/backtrace
+ ydb/library/yql/utils/log
+ ydb/library/yql/providers/common/proto
+ ydb/library/yql/providers/common/udf_resolve
+ ydb/library/yql/providers/solomon/gateway
+ ydb/library/yql/providers/solomon/provider
+ ydb/library/yql/core/url_preprocessing
+ ydb/library/yql/providers/yt/gateway/native
+ ydb/library/yql/providers/yt/lib/log
+ ydb/library/yql/providers/yt/lib/yt_download
+ ydb/library/yql/providers/yt/provider
+
+ yt/yql/plugin
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/yt/yql/plugin/plugin.cpp b/yt/yql/plugin/plugin.cpp
new file mode 100644
index 0000000000..5ebfffb796
--- /dev/null
+++ b/yt/yql/plugin/plugin.cpp
@@ -0,0 +1,18 @@
+#include "plugin.h"
+
+#include <iostream>
+
+namespace NYT::NYqlPlugin {
+
+////////////////////////////////////////////////////////////////////////////////
+
+Y_WEAK std::unique_ptr<IYqlPlugin> CreateYqlPlugin(TYqlPluginOptions& /*options*/) noexcept
+{
+ std::cerr << "No YQL plugin implementation is available; link against either "
+ << "yt/yql/plugin/native or yt/yql/plugin/dynamic" << std::endl;
+ exit(1);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NYqlPlugin
diff --git a/yt/yql/plugin/plugin.h b/yt/yql/plugin/plugin.h
new file mode 100644
index 0000000000..46a08ba5e7
--- /dev/null
+++ b/yt/yql/plugin/plugin.h
@@ -0,0 +1,65 @@
+#pragma once
+
+#include <util/generic/hash.h>
+#include <util/generic/string.h>
+
+#include <library/cpp/logger/log.h>
+
+#include <library/cpp/yt/yson_string/string.h>
+
+#include <optional>
+
+namespace NYT::NYqlPlugin {
+
+using namespace NYson;
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TYqlPluginOptions
+{
+public:
+ TString MRJobBinary = "./mrjob";
+ TString UdfDirectory;
+
+ //! Mapping cluster name -> proxy address.
+ THashMap<TString, TString> Clusters;
+ std::optional<TString> DefaultCluster;
+
+ TYsonString OperationAttributes;
+
+ TString YTTokenPath;
+
+ THolder<TLogBackend> LogBackend;
+
+ std::optional<TString> YqlPluginSharedLibrary;
+};
+
+struct TQueryResult
+{
+ std::optional<TString> YsonResult;
+ std::optional<TString> Plan;
+ std::optional<TString> Statistics;
+ std::optional<TString> TaskInfo;
+
+ //! YSON representation of a YT error.
+ std::optional<TString> YsonError;
+};
+
+//! This interface encapsulates YT <-> YQL integration.
+//! There are two major implementation: one of them is based
+//! on YQL code and another wraps the pure C bridge interface, which
+//! is implemented by a dynamic library.
+struct IYqlPlugin
+{
+ virtual TQueryResult Run(TString impersonationUser, TString queryText, TYsonString settings) noexcept = 0;
+
+ virtual ~IYqlPlugin() = default;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+Y_WEAK std::unique_ptr<IYqlPlugin> CreateYqlPlugin(TYqlPluginOptions& options) noexcept;
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NYqlPlugin
diff --git a/yt/yql/plugin/ya.make b/yt/yql/plugin/ya.make
new file mode 100644
index 0000000000..7046752106
--- /dev/null
+++ b/yt/yql/plugin/ya.make
@@ -0,0 +1,19 @@
+LIBRARY()
+
+SRCS(
+ plugin.cpp
+)
+
+END()
+
+RECURSE(
+ bridge
+)
+
+IF (NOT OPENSOURCE)
+ # We do not bring YQL with us into open source.
+ RECURSE(
+ dynamic
+ native
+ )
+ENDIF()