diff options
author | max42 <max42@yandex-team.com> | 2023-07-29 00:02:16 +0300 |
---|---|---|
committer | max42 <max42@yandex-team.com> | 2023-07-29 00:02:16 +0300 |
commit | 73b89de71748a21e102d27b9f3ed1bf658766cb5 (patch) | |
tree | 188bbd2d622fa91cdcbb1b6d6d77fbc84a0646f5 /yt/yql | |
parent | 528e321bcc2a2b67b53aeba58c3bd88305a141ee (diff) | |
download | ydb-73b89de71748a21e102d27b9f3ed1bf658766cb5.tar.gz |
YT-19210: expose YQL shared library for YT.
After this, a new target libyqlplugin.so appears. in open-source cmake build.
Diff in open-source YDB repo looks like the following: https://paste.yandex-team.ru/f302bdb4-7ef2-4362-91c7-6ca45f329264
Diffstat (limited to 'yt/yql')
31 files changed, 1562 insertions, 0 deletions
diff --git a/yt/yql/CMakeLists.txt b/yt/yql/CMakeLists.txt new file mode 100644 index 0000000000..8180a508fa --- /dev/null +++ b/yt/yql/CMakeLists.txt @@ -0,0 +1,9 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(plugin) diff --git a/yt/yql/plugin/CMakeLists.darwin-x86_64.txt b/yt/yql/plugin/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..e4e42a5389 --- /dev/null +++ b/yt/yql/plugin/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,19 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(dynamic) +add_subdirectory(native) + +add_library(yt-yql-plugin) +target_link_libraries(yt-yql-plugin PUBLIC + contrib-libs-cxxsupp + yutil +) +target_sources(yt-yql-plugin PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/plugin.cpp +) diff --git a/yt/yql/plugin/CMakeLists.linux-aarch64.txt b/yt/yql/plugin/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..3fb7f34a94 --- /dev/null +++ b/yt/yql/plugin/CMakeLists.linux-aarch64.txt @@ -0,0 +1,20 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(dynamic) +add_subdirectory(native) + +add_library(yt-yql-plugin) +target_link_libraries(yt-yql-plugin PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil +) +target_sources(yt-yql-plugin PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/plugin.cpp +) diff --git a/yt/yql/plugin/CMakeLists.linux-x86_64.txt b/yt/yql/plugin/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..3fb7f34a94 --- /dev/null +++ b/yt/yql/plugin/CMakeLists.linux-x86_64.txt @@ -0,0 +1,20 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(dynamic) +add_subdirectory(native) + +add_library(yt-yql-plugin) +target_link_libraries(yt-yql-plugin PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil +) +target_sources(yt-yql-plugin PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/plugin.cpp +) diff --git a/yt/yql/plugin/CMakeLists.txt b/yt/yql/plugin/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/yt/yql/plugin/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/yt/yql/plugin/CMakeLists.windows-x86_64.txt b/yt/yql/plugin/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..e4e42a5389 --- /dev/null +++ b/yt/yql/plugin/CMakeLists.windows-x86_64.txt @@ -0,0 +1,19 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(dynamic) +add_subdirectory(native) + +add_library(yt-yql-plugin) +target_link_libraries(yt-yql-plugin PUBLIC + contrib-libs-cxxsupp + yutil +) +target_sources(yt-yql-plugin PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/plugin.cpp +) diff --git a/yt/yql/plugin/bridge/interface.h b/yt/yql/plugin/bridge/interface.h new file mode 100644 index 0000000000..062adb7ea5 --- /dev/null +++ b/yt/yql/plugin/bridge/interface.h @@ -0,0 +1,74 @@ +#pragma once + +#include <unistd.h> + +//////////////////////////////////////////////////////////////////////////////// + +// This is a plain C version of an interface described in yt/yql/plugin/plugin.h. +// All strings without separate length field are assumed to be null-terminated. + +//////////////////////////////////////////////////////////////////////////////// + +struct TBridgeYqlPluginOptions +{ + const char* MRJobBinary; + const char* UdfDirectory; + + struct TBridgeCluster + { + const char* Cluster; + const char* Proxy; + }; + ssize_t ClusterCount; + TBridgeCluster* Clusters; + const char* DefaultCluster; + + const char* OperationAttributes; + + const char* YTTokenPath; + + // TODO(max42): passing C++ objects across shared libraries is incredibly + // fragile. This is a temporary mean until we come up with something more + // convenient; get rid of this ASAP. + using TLogBackendHolder = void; + TLogBackendHolder* LogBackend; +}; + +// Opaque type representing a YQL plugin. +using TBridgeYqlPlugin = void; + +using TFuncBridgeCreateYqlPlugin = TBridgeYqlPlugin*(const TBridgeYqlPluginOptions* options); +using TFuncBridgeFreeYqlPlugin = void(TBridgeYqlPlugin* plugin); + +//////////////////////////////////////////////////////////////////////////////// + +// TODO(max42): consider making structure an opaque type with accessors a-la +// const char* BridgeGetYsonResult(const TBridgeQueryResult*). This would remove the need +// to manually free string data. +struct TBridgeQueryResult +{ + const char* YsonResult = nullptr; + ssize_t YsonResultLength = 0; + const char* Plan = nullptr; + ssize_t PlanLength = 0; + const char* Statistics = nullptr; + ssize_t StatisticsLength = 0; + const char* TaskInfo = nullptr; + ssize_t TaskInfoLength = 0; + + const char* YsonError = nullptr; + ssize_t YsonErrorLength = 0; +}; + +using TFuncBridgeFreeQueryResult = void(TBridgeQueryResult* result); +using TFuncBridgeRun = TBridgeQueryResult*(TBridgeYqlPlugin* plugin, const char* impersonationUser, const char* queryText, const char* settings); + +//////////////////////////////////////////////////////////////////////////////// + +#define FOR_EACH_BRIDGE_INTERFACE_FUNCTION(XX) \ + XX(BridgeCreateYqlPlugin) \ + XX(BridgeFreeYqlPlugin) \ + XX(BridgeFreeQueryResult) \ + XX(BridgeRun) + +//////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yql/plugin/bridge/plugin.cpp b/yt/yql/plugin/bridge/plugin.cpp new file mode 100644 index 0000000000..52b9f92a01 --- /dev/null +++ b/yt/yql/plugin/bridge/plugin.cpp @@ -0,0 +1,120 @@ +#include "plugin.h" + +#include "interface.h" + +#include <yt/yql/plugin/plugin.h> +#include <util/system/dynlib.h> + +#include <vector> +#include <optional> + +namespace NYT::NYqlPlugin { +namespace NBridge { + +//////////////////////////////////////////////////////////////////////////////// + +class TDynamicYqlPlugin +{ +public: + TDynamicYqlPlugin(std::optional<TString> yqlPluginSharedLibrary) + { + const TString DefaultYqlPluginLibraryName = "./libyqlplugin.so"; + auto sharedLibraryPath = yqlPluginSharedLibrary.value_or(DefaultYqlPluginLibraryName); + Library_.Open(sharedLibraryPath.data()); + #define XX(function) function = reinterpret_cast<TFunc ## function*>(Library_.Sym(#function)); + FOR_EACH_BRIDGE_INTERFACE_FUNCTION(XX); + #undef XX + } + +protected: + #define XX(function) TFunc ## function* function; + FOR_EACH_BRIDGE_INTERFACE_FUNCTION(XX) + #undef XX + + TDynamicLibrary Library_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TYqlPlugin + : public TDynamicYqlPlugin + , public IYqlPlugin +{ +public: + explicit TYqlPlugin(TYqlPluginOptions& options) + : TDynamicYqlPlugin(options.YqlPluginSharedLibrary) + { + std::vector<TBridgeYqlPluginOptions::TBridgeCluster> bridgeClusters; + for (const auto& [cluster, proxy]: options.Clusters) { + bridgeClusters.push_back({ + .Cluster = cluster.data(), + .Proxy = proxy.data(), + }); + } + + const char* operationAttributes = options.OperationAttributes + ? options.OperationAttributes.ToString().data() + : nullptr; + + const char* defaultCluster = options.DefaultCluster + ? options.DefaultCluster->data() + : nullptr; + + TBridgeYqlPluginOptions bridgeOptions { + .MRJobBinary = options.MRJobBinary.data(), + .UdfDirectory = options.UdfDirectory.data(), + .ClusterCount = static_cast<int>(bridgeClusters.size()), + .Clusters = bridgeClusters.data(), + .DefaultCluster = defaultCluster, + .OperationAttributes = operationAttributes, + .YTTokenPath = options.YTTokenPath.data(), + .LogBackend = &options.LogBackend, + }; + + BridgePlugin_ = BridgeCreateYqlPlugin(&bridgeOptions); + } + + TQueryResult Run(TString impersonationUser, TString queryText, NYson::TYsonString settings) noexcept override + { + const char* settingsData = settings ? settings.ToString().data() : nullptr; + auto* bridgeQueryResult = BridgeRun(BridgePlugin_, impersonationUser.data(), queryText.data(), settingsData); + auto toString = [] (const char* str, size_t strLength) -> std::optional<TString> { + if (!str) { + return std::nullopt; + } + return TString(str, strLength); + }; + TQueryResult queryResult = { + .YsonResult = toString(bridgeQueryResult->YsonResult, bridgeQueryResult->YsonResultLength), + .Plan = toString(bridgeQueryResult->Plan, bridgeQueryResult->PlanLength), + .Statistics = toString(bridgeQueryResult->Statistics, bridgeQueryResult->StatisticsLength), + .TaskInfo = toString(bridgeQueryResult->TaskInfo, bridgeQueryResult->TaskInfoLength), + .YsonError = toString(bridgeQueryResult->YsonError, bridgeQueryResult->YsonErrorLength), + }; + BridgeFreeQueryResult(bridgeQueryResult); + return queryResult; + } + + ~TYqlPlugin() override + { + BridgeFreeYqlPlugin(BridgePlugin_); + } + +private: + TBridgeYqlPlugin* BridgePlugin_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NBridge + +//////////////////////////////////////////////////////////////////////////////// + +std::unique_ptr<IYqlPlugin> CreateYqlPlugin(TYqlPluginOptions& options) noexcept +{ + return std::make_unique<NBridge::TYqlPlugin>(options); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYqlPlugin::NBridge diff --git a/yt/yql/plugin/bridge/plugin.h b/yt/yql/plugin/bridge/plugin.h new file mode 100644 index 0000000000..a80528e700 --- /dev/null +++ b/yt/yql/plugin/bridge/plugin.h @@ -0,0 +1,13 @@ +#pragma once + +#include <yt/yql/plugin/plugin.h> + +namespace NYT::NYqlPlugin { + +//////////////////////////////////////////////////////////////////////////////// + +std::unique_ptr<IYqlPlugin> CreateYqlPlugin(TYqlPluginOptions& options) noexcept; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYqlPlugin diff --git a/yt/yql/plugin/bridge/ya.make b/yt/yql/plugin/bridge/ya.make new file mode 100644 index 0000000000..93425c5284 --- /dev/null +++ b/yt/yql/plugin/bridge/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +SRCS( + GLOBAL plugin.cpp +) + +PEERDIR( + yt/yql/plugin +) + +END() diff --git a/yt/yql/plugin/dynamic/CMakeLists.darwin-x86_64.txt b/yt/yql/plugin/dynamic/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..bb76915330 --- /dev/null +++ b/yt/yql/plugin/dynamic/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,31 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_shared_library(yqlplugin) +target_link_libraries(yqlplugin PUBLIC + contrib-libs-cxxsupp + yutil + yql-plugin-native +) +target_link_options(yqlplugin PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -undefined + dynamic_lookup + -fPIC + -framework + CoreFoundation +) +target_sources(yqlplugin PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/dynamic/impl.cpp +) +use_export_script(yqlplugin + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/dynamic/dylib.exports +) +vcs_info(yqlplugin) diff --git a/yt/yql/plugin/dynamic/CMakeLists.linux-aarch64.txt b/yt/yql/plugin/dynamic/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..63edf774f2 --- /dev/null +++ b/yt/yql/plugin/dynamic/CMakeLists.linux-aarch64.txt @@ -0,0 +1,35 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_shared_library(yqlplugin) +target_link_libraries(yqlplugin PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yql-plugin-native +) +target_link_options(yqlplugin PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -Wl,-z,notext + -fPIC + -lpthread + -lrt + -ldl + -lutil +) +target_sources(yqlplugin PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/dynamic/impl.cpp +) +use_export_script(yqlplugin + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/dynamic/dylib.exports +) +vcs_info(yqlplugin) diff --git a/yt/yql/plugin/dynamic/CMakeLists.linux-x86_64.txt b/yt/yql/plugin/dynamic/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..63edf774f2 --- /dev/null +++ b/yt/yql/plugin/dynamic/CMakeLists.linux-x86_64.txt @@ -0,0 +1,35 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_shared_library(yqlplugin) +target_link_libraries(yqlplugin PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yql-plugin-native +) +target_link_options(yqlplugin PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -Wl,-z,notext + -fPIC + -lpthread + -lrt + -ldl + -lutil +) +target_sources(yqlplugin PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/dynamic/impl.cpp +) +use_export_script(yqlplugin + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/dynamic/dylib.exports +) +vcs_info(yqlplugin) diff --git a/yt/yql/plugin/dynamic/CMakeLists.txt b/yt/yql/plugin/dynamic/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/yt/yql/plugin/dynamic/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/yt/yql/plugin/dynamic/CMakeLists.windows-x86_64.txt b/yt/yql/plugin/dynamic/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..2814417000 --- /dev/null +++ b/yt/yql/plugin/dynamic/CMakeLists.windows-x86_64.txt @@ -0,0 +1,22 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_shared_library(yqlplugin) +target_link_libraries(yqlplugin PUBLIC + contrib-libs-cxxsupp + yutil + yql-plugin-native +) +target_sources(yqlplugin PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/dynamic/impl.cpp +) +use_export_script(yqlplugin + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/dynamic/dylib.exports +) +vcs_info(yqlplugin) diff --git a/yt/yql/plugin/dynamic/dylib.exports b/yt/yql/plugin/dynamic/dylib.exports new file mode 100644 index 0000000000..ec1c95cb11 --- /dev/null +++ b/yt/yql/plugin/dynamic/dylib.exports @@ -0,0 +1,12 @@ +# YT <-> YQL bridge. +BridgeCreateYqlPlugin +BridgeFreeYqlPlugin +BridgeFreeQueryResult +BridgeRun + +# YQL <-> YQL UDFs interface. +UdfAllocateWithSize +UdfFreeWithSize +UdfRegisterObject +UdfTerminate +UdfUnregisterObject diff --git a/yt/yql/plugin/dynamic/impl.cpp b/yt/yql/plugin/dynamic/impl.cpp new file mode 100644 index 0000000000..e39897c97e --- /dev/null +++ b/yt/yql/plugin/dynamic/impl.cpp @@ -0,0 +1,93 @@ +#include <yt/yql/plugin/bridge/interface.h> +#include <yt/yql/plugin/native/plugin.h> + +#include <type_traits> + +using namespace NYT::NYqlPlugin; +using namespace NYT::NYson; + +extern "C" { + +//////////////////////////////////////////////////////////////////////////////// + +TBridgeYqlPlugin* BridgeCreateYqlPlugin(const TBridgeYqlPluginOptions* bridgeOptions) +{ + THashMap<TString, TString> clusters; + for (auto clusterIndex = 0; clusterIndex < bridgeOptions->ClusterCount; ++clusterIndex) { + const auto& Cluster = bridgeOptions->Clusters[clusterIndex]; + clusters[Cluster.Cluster] = Cluster.Proxy; + } + + TYsonString operationAttributes = bridgeOptions->OperationAttributes + ? TYsonString(TString(bridgeOptions->OperationAttributes)) + : TYsonString{}; + + TYqlPluginOptions options{ + .MRJobBinary = TString(bridgeOptions->MRJobBinary), + .UdfDirectory = TString(bridgeOptions->UdfDirectory), + .Clusters = std::move(clusters), + .DefaultCluster = std::optional<TString>(bridgeOptions->DefaultCluster), + .OperationAttributes = operationAttributes, + .YTTokenPath = TString(bridgeOptions->YTTokenPath), + .LogBackend = std::move(*reinterpret_cast<THolder<TLogBackend>*>(bridgeOptions->LogBackend)), + }; + auto nativePlugin = CreateYqlPlugin(options); + return nativePlugin.release(); +} + +void BridgeFreeYqlPlugin(TBridgeYqlPlugin* plugin) +{ + auto* nativePlugin = reinterpret_cast<IYqlPlugin*>(plugin); + delete nativePlugin; +} + +void BridgeFreeQueryResult(TBridgeQueryResult* result) +{ + delete result->TaskInfo; + delete result->Statistics; + delete result->Plan; + delete result->YsonResult; + delete result->YsonError; + delete result; +} + +TBridgeQueryResult* BridgeRun(TBridgeYqlPlugin* plugin, const char* impersonationUser, const char* queryText, const char* settings) +{ + static const TYsonString EmptyMap = TYsonString(TString("{}")); + + auto* nativePlugin = reinterpret_cast<IYqlPlugin*>(plugin); + auto* bridgeResult = new TBridgeQueryResult; + + auto fillString = [] (const char*& str, ssize_t& strLength, const std::optional<TString>& original) { + if (!original) { + str = nullptr; + strLength = 0; + return; + } + char* copy = new char[original->size() + 1]; + memcpy(copy, original->data(), original->size() + 1); + str = copy; + strLength = original->size(); + }; + + auto result = nativePlugin->Run(TString(impersonationUser), TString(queryText), settings ? TYsonString(TString(settings)) : EmptyMap); + fillString(bridgeResult->YsonResult, bridgeResult->YsonResultLength, result.YsonResult); + fillString(bridgeResult->Plan, bridgeResult->PlanLength, result.Plan); + fillString(bridgeResult->Statistics, bridgeResult->StatisticsLength, result.Statistics); + fillString(bridgeResult->TaskInfo, bridgeResult->TaskInfoLength, result.TaskInfo); + fillString(bridgeResult->YsonError, bridgeResult->YsonErrorLength, result.YsonError); + + return bridgeResult; +} + +//////////////////////////////////////////////////////////////////////////////// + +// Validate that the all functions from the bridge interface are implemented with proper signatures. + +#define XX(function) static_assert(std::is_same_v<decltype(&(function)), TFunc ## function*>); +FOR_EACH_BRIDGE_INTERFACE_FUNCTION(XX) +#undef XX + +//////////////////////////////////////////////////////////////////////////////// + +} // extern "C" diff --git a/yt/yql/plugin/dynamic/ya.make b/yt/yql/plugin/dynamic/ya.make new file mode 100644 index 0000000000..c91996acec --- /dev/null +++ b/yt/yql/plugin/dynamic/ya.make @@ -0,0 +1,15 @@ +DLL(yqlplugin 1 0) + +INCLUDE(${ARCADIA_ROOT}/yt/opensource.inc) + +EXPORTS_SCRIPT(dylib.exports) + +SRCS( + impl.cpp +) + +PEERDIR( + yt/yql/plugin/native +) + +END() diff --git a/yt/yql/plugin/native/CMakeLists.darwin-x86_64.txt b/yt/yql/plugin/native/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..753bac36c8 --- /dev/null +++ b/yt/yql/plugin/native/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,92 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yql-plugin-native) +target_compile_options(yql-plugin-native PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(yql-plugin-native PUBLIC + contrib-libs-cxxsupp + yutil + contrib-libs-protobuf + library-cpp-resource + library-cpp-yson + cpp-yson-node + cpp-mapreduce-client + cpp-mapreduce-common + library-yql-ast + yql-sql-pg + yql-parser-pg_wrapper + yql-core-facade + yql-core-file_storage + core-file_storage-proto + core-file_storage-http_download + core-services-mounts + yql-core-user_data + library-yql-minikql + library-yql-protos + udf-service-exception_policy + yql-utils-backtrace + yql-utils-log + providers-common-proto + providers-common-udf_resolve + providers-solomon-gateway + providers-solomon-provider + yql-core-url_preprocessing + yt-gateway-native + yt-lib-log + yt-lib-yt_download + providers-yt-provider + yt-yql-plugin +) +target_sources(yql-plugin-native PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/native/error_helpers.cpp +) + +add_global_library_for(yql-plugin-native.global yql-plugin-native) +target_compile_options(yql-plugin-native.global PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(yql-plugin-native.global PUBLIC + contrib-libs-cxxsupp + yutil + contrib-libs-protobuf + library-cpp-resource + library-cpp-yson + cpp-yson-node + cpp-mapreduce-client + cpp-mapreduce-common + library-yql-ast + yql-sql-pg + yql-parser-pg_wrapper + yql-core-facade + yql-core-file_storage + core-file_storage-proto + core-file_storage-http_download + core-services-mounts + yql-core-user_data + library-yql-minikql + library-yql-protos + udf-service-exception_policy + yql-utils-backtrace + yql-utils-log + providers-common-proto + providers-common-udf_resolve + providers-solomon-gateway + providers-solomon-provider + yql-core-url_preprocessing + yt-gateway-native + yt-lib-log + yt-lib-yt_download + providers-yt-provider + yt-yql-plugin +) +target_sources(yql-plugin-native.global PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/native/plugin.cpp +) diff --git a/yt/yql/plugin/native/CMakeLists.linux-aarch64.txt b/yt/yql/plugin/native/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..348d9ca192 --- /dev/null +++ b/yt/yql/plugin/native/CMakeLists.linux-aarch64.txt @@ -0,0 +1,94 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yql-plugin-native) +target_compile_options(yql-plugin-native PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(yql-plugin-native PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + contrib-libs-protobuf + library-cpp-resource + library-cpp-yson + cpp-yson-node + cpp-mapreduce-client + cpp-mapreduce-common + library-yql-ast + yql-sql-pg + yql-parser-pg_wrapper + yql-core-facade + yql-core-file_storage + core-file_storage-proto + core-file_storage-http_download + core-services-mounts + yql-core-user_data + library-yql-minikql + library-yql-protos + udf-service-exception_policy + yql-utils-backtrace + yql-utils-log + providers-common-proto + providers-common-udf_resolve + providers-solomon-gateway + providers-solomon-provider + yql-core-url_preprocessing + yt-gateway-native + yt-lib-log + yt-lib-yt_download + providers-yt-provider + yt-yql-plugin +) +target_sources(yql-plugin-native PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/native/error_helpers.cpp +) + +add_global_library_for(yql-plugin-native.global yql-plugin-native) +target_compile_options(yql-plugin-native.global PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(yql-plugin-native.global PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + contrib-libs-protobuf + library-cpp-resource + library-cpp-yson + cpp-yson-node + cpp-mapreduce-client + cpp-mapreduce-common + library-yql-ast + yql-sql-pg + yql-parser-pg_wrapper + yql-core-facade + yql-core-file_storage + core-file_storage-proto + core-file_storage-http_download + core-services-mounts + yql-core-user_data + library-yql-minikql + library-yql-protos + udf-service-exception_policy + yql-utils-backtrace + yql-utils-log + providers-common-proto + providers-common-udf_resolve + providers-solomon-gateway + providers-solomon-provider + yql-core-url_preprocessing + yt-gateway-native + yt-lib-log + yt-lib-yt_download + providers-yt-provider + yt-yql-plugin +) +target_sources(yql-plugin-native.global PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/native/plugin.cpp +) diff --git a/yt/yql/plugin/native/CMakeLists.linux-x86_64.txt b/yt/yql/plugin/native/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..348d9ca192 --- /dev/null +++ b/yt/yql/plugin/native/CMakeLists.linux-x86_64.txt @@ -0,0 +1,94 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yql-plugin-native) +target_compile_options(yql-plugin-native PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(yql-plugin-native PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + contrib-libs-protobuf + library-cpp-resource + library-cpp-yson + cpp-yson-node + cpp-mapreduce-client + cpp-mapreduce-common + library-yql-ast + yql-sql-pg + yql-parser-pg_wrapper + yql-core-facade + yql-core-file_storage + core-file_storage-proto + core-file_storage-http_download + core-services-mounts + yql-core-user_data + library-yql-minikql + library-yql-protos + udf-service-exception_policy + yql-utils-backtrace + yql-utils-log + providers-common-proto + providers-common-udf_resolve + providers-solomon-gateway + providers-solomon-provider + yql-core-url_preprocessing + yt-gateway-native + yt-lib-log + yt-lib-yt_download + providers-yt-provider + yt-yql-plugin +) +target_sources(yql-plugin-native PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/native/error_helpers.cpp +) + +add_global_library_for(yql-plugin-native.global yql-plugin-native) +target_compile_options(yql-plugin-native.global PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(yql-plugin-native.global PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + contrib-libs-protobuf + library-cpp-resource + library-cpp-yson + cpp-yson-node + cpp-mapreduce-client + cpp-mapreduce-common + library-yql-ast + yql-sql-pg + yql-parser-pg_wrapper + yql-core-facade + yql-core-file_storage + core-file_storage-proto + core-file_storage-http_download + core-services-mounts + yql-core-user_data + library-yql-minikql + library-yql-protos + udf-service-exception_policy + yql-utils-backtrace + yql-utils-log + providers-common-proto + providers-common-udf_resolve + providers-solomon-gateway + providers-solomon-provider + yql-core-url_preprocessing + yt-gateway-native + yt-lib-log + yt-lib-yt_download + providers-yt-provider + yt-yql-plugin +) +target_sources(yql-plugin-native.global PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/native/plugin.cpp +) diff --git a/yt/yql/plugin/native/CMakeLists.txt b/yt/yql/plugin/native/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/yt/yql/plugin/native/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/yt/yql/plugin/native/CMakeLists.windows-x86_64.txt b/yt/yql/plugin/native/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..753bac36c8 --- /dev/null +++ b/yt/yql/plugin/native/CMakeLists.windows-x86_64.txt @@ -0,0 +1,92 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yql-plugin-native) +target_compile_options(yql-plugin-native PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(yql-plugin-native PUBLIC + contrib-libs-cxxsupp + yutil + contrib-libs-protobuf + library-cpp-resource + library-cpp-yson + cpp-yson-node + cpp-mapreduce-client + cpp-mapreduce-common + library-yql-ast + yql-sql-pg + yql-parser-pg_wrapper + yql-core-facade + yql-core-file_storage + core-file_storage-proto + core-file_storage-http_download + core-services-mounts + yql-core-user_data + library-yql-minikql + library-yql-protos + udf-service-exception_policy + yql-utils-backtrace + yql-utils-log + providers-common-proto + providers-common-udf_resolve + providers-solomon-gateway + providers-solomon-provider + yql-core-url_preprocessing + yt-gateway-native + yt-lib-log + yt-lib-yt_download + providers-yt-provider + yt-yql-plugin +) +target_sources(yql-plugin-native PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/native/error_helpers.cpp +) + +add_global_library_for(yql-plugin-native.global yql-plugin-native) +target_compile_options(yql-plugin-native.global PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(yql-plugin-native.global PUBLIC + contrib-libs-cxxsupp + yutil + contrib-libs-protobuf + library-cpp-resource + library-cpp-yson + cpp-yson-node + cpp-mapreduce-client + cpp-mapreduce-common + library-yql-ast + yql-sql-pg + yql-parser-pg_wrapper + yql-core-facade + yql-core-file_storage + core-file_storage-proto + core-file_storage-http_download + core-services-mounts + yql-core-user_data + library-yql-minikql + library-yql-protos + udf-service-exception_policy + yql-utils-backtrace + yql-utils-log + providers-common-proto + providers-common-udf_resolve + providers-solomon-gateway + providers-solomon-provider + yql-core-url_preprocessing + yt-gateway-native + yt-lib-log + yt-lib-yt_download + providers-yt-provider + yt-yql-plugin +) +target_sources(yql-plugin-native.global PRIVATE + ${CMAKE_SOURCE_DIR}/yt/yql/plugin/native/plugin.cpp +) diff --git a/yt/yql/plugin/native/error_helpers.cpp b/yt/yql/plugin/native/error_helpers.cpp new file mode 100644 index 0000000000..23b9fa5cba --- /dev/null +++ b/yt/yql/plugin/native/error_helpers.cpp @@ -0,0 +1,121 @@ +#include "error_helpers.h" + +#include <library/cpp/yson/writer.h> + +#include <ydb/library/yql/public/issue/yql_issue_id.h> + +#include <ydb/library/yql/core/issue/protos/issue_id.pb.h> + +namespace NYT::NYqlPlugin { + +//////////////////////////////////////////////////////////////////////////////// + +const int IssueToErrorCodesShift = 30000; + +//////////////////////////////////////////////////////////////////////////////// + +TString ExceptionToYtErrorYson(const std::exception& exception) +{ + TStringStream yson; + ::NYson::TYsonWriter writer(&yson); + + writer.OnBeginMap(); + writer.OnKeyedItem("code"); + writer.OnInt64Scalar(1); // Generic error + writer.OnKeyedItem("message"); + writer.OnStringScalar(exception.what()); + writer.OnKeyedItem("attributes"); + writer.OnBeginMap(); + writer.OnEndMap(); + + writer.OnEndMap(); + + return yson.Str(); +} + +TString IssuesToYtErrorYson(const NYql::TIssues& issues) +{ + TStringStream yson; + ::NYson::TYsonWriter writer(&yson); + + auto serializePosition = [&] (const NYql::TPosition& position) { + writer.OnBeginMap(); + { + writer.OnKeyedItem("column"); + writer.OnInt64Scalar(position.Column); + + writer.OnKeyedItem("row"); + writer.OnInt64Scalar(position.Row); + + if (!position.File.empty()) { + writer.OnKeyedItem("file"); + writer.OnStringScalar(position.File); + } + } + writer.OnEndMap(); + }; + + auto fn = [&] (const NYql::TIssue& issue, ui16 /*level*/) { + writer.OnListItem(); + writer.OnBeginMap(); + { + writer.OnKeyedItem("code"); + + NYql::TIssueCode code = IssueToErrorCodesShift + issue.GetCode(); + writer.OnInt64Scalar(code); + + writer.OnKeyedItem("message"); + writer.OnStringScalar(issue.GetMessage()); + + writer.OnKeyedItem("attributes"); + writer.OnBeginMap(); + { + if (issue.Range().Position) { + writer.OnKeyedItem("start_position"); + serializePosition(issue.Range().Position); + } + + if (issue.Range().EndPosition) { + writer.OnKeyedItem("end_position"); + serializePosition(issue.Range().EndPosition); + } + + writer.OnKeyedItem("yql_status"); + writer.OnStringScalar(NYql::IssueCodeToString<NYql::TIssuesIds>(issue.GetCode())); + + writer.OnKeyedItem("severity"); + writer.OnStringScalar(SeverityToString(issue.GetSeverity())); + } + writer.OnEndMap(); + + writer.OnKeyedItem("inner_errors"); + writer.OnBeginList(); + } + }; + + auto afterChildrenFn = [&] (const NYql::TIssue& /*issue*/, ui16 /*level*/) { + writer.OnEndList(); + writer.OnEndMap(); + }; + + writer.OnBeginMap(); + writer.OnKeyedItem("message"); + writer.OnStringScalar("There are some issues"); + writer.OnKeyedItem("code"); + writer.OnInt64Scalar(1); + writer.OnKeyedItem("inner_errors"); + writer.OnBeginList(); + for (const auto& issue : issues) { + WalkThroughIssues(issue, /*leafOnly*/ false, fn, afterChildrenFn); + } + writer.OnEndList(); + writer.OnKeyedItem("attributes"); + writer.OnBeginMap(); + writer.OnEndMap(); + writer.OnEndMap(); + return yson.Str(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYqlPLugin diff --git a/yt/yql/plugin/native/error_helpers.h b/yt/yql/plugin/native/error_helpers.h new file mode 100644 index 0000000000..1905502aa8 --- /dev/null +++ b/yt/yql/plugin/native/error_helpers.h @@ -0,0 +1,17 @@ +#pragma once + +#include <util/generic/string.h> + +#include <ydb/library/yql/public/issue/yql_issue.h> + +namespace NYT::NYqlPlugin { + +//////////////////////////////////////////////////////////////////////////////// + +TString IssuesToYtErrorYson(const NYql::TIssues& issues); + +TString ExceptionToYtErrorYson(const std::exception& exception); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYqlPlugin diff --git a/yt/yql/plugin/native/plugin.cpp b/yt/yql/plugin/native/plugin.cpp new file mode 100644 index 0000000000..086f63a2f9 --- /dev/null +++ b/yt/yql/plugin/native/plugin.cpp @@ -0,0 +1,293 @@ +#include "plugin.h" + +#include "error_helpers.h" + +#include <ydb/library/yql/providers/yt/lib/log/yt_logger.h> +#include <ydb/library/yql/providers/yt/lib/yt_download/yt_download.h> +#include <ydb/library/yql/providers/yt/gateway/native/yql_yt_native.h> +#include <ydb/library/yql/providers/yt/provider/yql_yt_provider.h> + +#include <ydb/library/yql/core/url_preprocessing/url_preprocessing.h> + +#include <ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.h> +#include "ydb/library/yql/providers/common/proto/gateways_config.pb.h" +#include <ydb/library/yql/providers/common/provider/yql_provider_names.h> + +#include <ydb/library/yql/ast/yql_expr.h> +#include <ydb/library/yql/minikql/mkql_function_registry.h> +#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> +#include <ydb/library/yql/core/facade/yql_facade.h> +#include <ydb/library/yql/core/file_storage/file_storage.h> +#include "ydb/library/yql/core/file_storage/proto/file_storage.pb.h" +#include <ydb/library/yql/core/services/mounts/yql_mounts.h> +#include <ydb/library/yql/utils/log/log.h> +#include <ydb/library/yql/utils/backtrace/backtrace.h> + +#include <yt/cpp/mapreduce/interface/config.h> +#include <yt/cpp/mapreduce/interface/logging/logger.h> + +#include <library/cpp/yson/node/node_io.h> + +#include <library/cpp/yson/parser.h> +#include <library/cpp/yson/writer.h> + +#include <library/cpp/resource/resource.h> +#include <library/cpp/digest/md5/md5.h> + +#include <util/folder/path.h> +#include <util/stream/file.h> +#include <util/string/builder.h> +#include <util/system/fs.h> +#include <util/system/user.h> + +namespace NYT::NYqlPlugin { +namespace NNative { + +using namespace NYson; + +//////////////////////////////////////////////////////////////////////////////// + +class TYqlPlugin + : public IYqlPlugin +{ +public: + TYqlPlugin(TYqlPluginOptions& options) + { + try { + NYql::NLog::InitLogger(std::move(options.LogBackend)); + + auto& logger = NYql::NLog::YqlLogger(); + + logger.SetDefaultPriority(ELogPriority::TLOG_DEBUG); + for (int i = 0; i < NYql::NLog::EComponentHelpers::ToInt(NYql::NLog::EComponent::MaxValue); ++i) { + logger.SetComponentLevel((NYql::NLog::EComponent) i, NYql::NLog::ELevel::DEBUG); + } + + NYql::SetYtLoggerGlobalBackend(NYT::ILogger::ELevel::DEBUG); + if (NYT::TConfig::Get()->Prefix.empty()) { + NYT::TConfig::Get()->Prefix = "//"; + } + + auto yqlCoreFlags = GatewaysConfig_.GetYqlCore() + .GetFlags(); + + auto ytConfig = GatewaysConfig_.MutableYt(); + if (!ytConfig->HasExecuteUdfLocallyIfPossible()) { + ytConfig->SetExecuteUdfLocallyIfPossible(true); + } + + ytConfig->SetYtLogLevel(NYql::EYtLogLevel::YL_DEBUG); + ytConfig->SetMrJobBin(options.MRJobBinary); + ytConfig->SetMrJobBinMd5(MD5::File(options.MRJobBinary)); + + ytConfig->ClearMrJobUdfsDir(); + + for (const auto& [cluster, address]: options.Clusters) { + auto item = ytConfig->AddClusterMapping(); + item->SetName(cluster); + item->SetCluster(address); + if (cluster == options.DefaultCluster) { + item->SetDefault(true); + } + + Clusters_.insert({item->GetName(), TString(NYql::YtProviderName)}); + } + DefaultCluster_ = options.DefaultCluster; + + NYql::TFileStorageConfig fileStorageConfig; + fileStorageConfig.SetMaxSizeMb(1 << 14); + FileStorage_ = WithAsync(CreateFileStorage(fileStorageConfig, {MakeYtDownloader(fileStorageConfig)})); + + FuncRegistry_ = NKikimr::NMiniKQL::CreateFunctionRegistry( + NKikimr::NMiniKQL::CreateBuiltinRegistry())->Clone(); + + const NKikimr::NMiniKQL::TUdfModuleRemappings emptyRemappings; + + FuncRegistry_->SetBackTraceCallback(&NYql::NBacktrace::KikimrBackTrace); + + NKikimr::NMiniKQL::TUdfModulePathsMap systemModules; + + TVector<TString> udfPaths; + NKikimr::NMiniKQL::FindUdfsInDir(options.UdfDirectory, &udfPaths); + for (const auto& path: udfPaths) { + // Skip YQL plugin shared library itself, it is not a UDF. + if (path.EndsWith("libyqlplugin.so")) { + continue; + } + FuncRegistry_->LoadUdfs(path, emptyRemappings, 0); + } + + for (auto& m: FuncRegistry_->GetAllModuleNames()) { + TMaybe<TString> path = FuncRegistry_->FindUdfPath(m); + if (!path) { + YQL_LOG(FATAL) << "Unable to detect UDF path for module " << m; + exit(1); + } + systemModules.emplace(m, *path); + } + + FuncRegistry_->SetSystemModulePaths(systemModules); + + NYql::TUserDataTable userDataTable = GetYqlModuleResolver(ExprContext_, ModuleResolver_, {}, Clusters_, {}); + + if (!userDataTable) { + TStringStream err; + ExprContext_.IssueManager + .GetIssues() + .PrintTo(err); + YQL_LOG(FATAL) << "Failed to compile modules:\n" + << err.Str(); + exit(1); + } + + OperationAttributes_ = options.OperationAttributes; + + TVector<NYql::TDataProviderInitializer> dataProvidersInit; + + NYql::TYtNativeServices ytServices; + ytServices.FunctionRegistry = FuncRegistry_.Get(); + ytServices.FileStorage = FileStorage_; + ytServices.Config = std::make_shared<NYql::TYtGatewayConfig>(*ytConfig); + auto ytNativeGateway = CreateYtNativeGateway(ytServices); + dataProvidersInit.push_back(GetYtNativeDataProviderInitializer(ytNativeGateway)); + + ProgramFactory_ = std::make_unique<NYql::TProgramFactory>( + false, FuncRegistry_.Get(), ExprContext_.NextUniqueId, dataProvidersInit, "embedded"); + YTTokenPath_ = options.YTTokenPath; + ProgramFactory_->AddUserDataTable(userDataTable); + ProgramFactory_->SetModules(ModuleResolver_); + ProgramFactory_->SetUdfResolver(NYql::NCommon::CreateSimpleUdfResolver(FuncRegistry_.Get(), FileStorage_)); + ProgramFactory_->SetGatewaysConfig(&GatewaysConfig_); + ProgramFactory_->SetFileStorage(FileStorage_); + ProgramFactory_->SetUrlPreprocessing(MakeIntrusive<NYql::TUrlPreprocessing>(GatewaysConfig_)); + } catch (const std::exception& ex) { + YQL_LOG(FATAL) << "Unexpected exception while initializing YQL plugin: " << ex.what(); + exit(1); + } + YQL_LOG(INFO) << "YQL plugin initialized"; + } + + TQueryResult GuardedRun(TString impersonationUser, TString queryText, TYsonString settings) + { + auto credentials = MakeIntrusive<NYql::TCredentials>(); + if (YTTokenPath_) { + TFsPath path(YTTokenPath_); + auto token = TIFStream(path).ReadAll(); + + credentials->AddCredential("default_yt", NYql::TCredential("yt", "", token)); + } + + credentials->AddCredential("impersonation_user_yt", NYql::TCredential("yt", "", impersonationUser)); + ProgramFactory_->SetCredentials(credentials); + + auto program = ProgramFactory_->Create("-memory-", queryText); + program->SetOperationAttrsYson(PatchQueryAttributes(OperationAttributes_, settings)); + + NSQLTranslation::TTranslationSettings sqlSettings; + sqlSettings.ClusterMapping = Clusters_; + sqlSettings.ModuleMapping = Modules_; + if (DefaultCluster_) { + sqlSettings.DefaultCluster = *DefaultCluster_; + } + sqlSettings.SyntaxVersion = 1; + sqlSettings.V0Behavior = NSQLTranslation::EV0Behavior::Disable; + + if (!program->ParseSql(sqlSettings)) { + return TQueryResult{ + .YsonError = IssuesToYtErrorYson(program->Issues()), + }; + } + + if (!program->Compile(GetUsername())) { + return TQueryResult{ + .YsonError = IssuesToYtErrorYson(program->Issues()), + }; + } + + NYql::TProgram::TStatus status = NYql::TProgram::TStatus::Error; + status = program->Run(GetUsername(), nullptr, nullptr, nullptr); + + if (status == NYql::TProgram::TStatus::Error) { + return TQueryResult{ + .YsonError = IssuesToYtErrorYson(program->Issues()), + }; + } + + TStringStream result; + if (program->HasResults()) { + ::NYson::TYsonWriter yson(&result, EYsonFormat::Binary); + yson.OnBeginList(); + for (const auto& result: program->Results()) { + yson.OnListItem(); + yson.OnRaw(result); + } + yson.OnEndList(); + } + + auto maybeToOptional = [] (const TMaybe<TString>& maybeStr) -> std::optional<TString> { + if (!maybeStr) { + return std::nullopt; + } + return *maybeStr; + }; + + return { + .YsonResult = result.Empty() ? std::nullopt : std::make_optional(result.Str()), + .Plan = maybeToOptional(program->GetQueryPlan()), + .Statistics = maybeToOptional(program->GetStatistics()), + .TaskInfo = maybeToOptional(program->GetTasksInfo()), + }; + } + + TQueryResult Run(TString impersonationUser, TString queryText, TYsonString settings) noexcept override + { + try { + return GuardedRun(impersonationUser, queryText, settings); + } catch (const std::exception& ex) { + return TQueryResult{ + .YsonError = ExceptionToYtErrorYson(ex), + }; + } + } + +private: + NYql::TFileStoragePtr FileStorage_; + NYql::TExprContext ExprContext_; + ::TIntrusivePtr<NKikimr::NMiniKQL::IMutableFunctionRegistry> FuncRegistry_; + NYql::IModuleResolver::TPtr ModuleResolver_; + NYql::TGatewaysConfig GatewaysConfig_; + std::unique_ptr<NYql::TProgramFactory> ProgramFactory_; + TString YTTokenPath_; + THashMap<TString, TString> Clusters_; + std::optional<TString> DefaultCluster_; + THashMap<TString, TString> Modules_; + THashSet<TString> Libraries_; + TYsonString OperationAttributes_; + + TString PatchQueryAttributes(TYsonString configAttributes, TYsonString querySettings) + { + NYT::TNode querySettingsMap = NodeFromYsonString(querySettings.ToString()); + NYT::TNode resultAttributesMap = NodeFromYsonString(configAttributes.ToString()); + + for (const auto& item: querySettingsMap.AsMap()) { + resultAttributesMap[item.first] = item.second; + } + + return NodeToYsonString(resultAttributesMap); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NNative + +//////////////////////////////////////////////////////////////////////////////// + +std::unique_ptr<IYqlPlugin> CreateYqlPlugin(TYqlPluginOptions& options) noexcept +{ + return std::make_unique<NNative::TYqlPlugin>(options); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYqlPlugin diff --git a/yt/yql/plugin/native/plugin.h b/yt/yql/plugin/native/plugin.h new file mode 100644 index 0000000000..53bb7119ce --- /dev/null +++ b/yt/yql/plugin/native/plugin.h @@ -0,0 +1,14 @@ +#pragma once + +#include <yt/yql/plugin/plugin.h> + + +namespace NYT::NYqlPlugin { + +//////////////////////////////////////////////////////////////////////////////// + +std::unique_ptr<IYqlPlugin> CreateYqlPlugin(TYqlPluginOptions& options) noexcept; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYqlPlugin diff --git a/yt/yql/plugin/native/ya.make b/yt/yql/plugin/native/ya.make new file mode 100644 index 0000000000..36d562a4d2 --- /dev/null +++ b/yt/yql/plugin/native/ya.make @@ -0,0 +1,44 @@ +LIBRARY() + +SRCS( + GLOBAL plugin.cpp + error_helpers.cpp +) + +PEERDIR( + contrib/libs/protobuf + library/cpp/resource + library/cpp/yson + library/cpp/yson/node + yt/cpp/mapreduce/client + yt/cpp/mapreduce/common + ydb/library/yql/ast + ydb/library/yql/sql/pg + ydb/library/yql/parser/pg_wrapper + ydb/library/yql/core/facade + ydb/library/yql/core/file_storage + ydb/library/yql/core/file_storage/proto + ydb/library/yql/core/file_storage/http_download + ydb/library/yql/core/services/mounts + ydb/library/yql/core/user_data + ydb/library/yql/minikql + ydb/library/yql/protos + ydb/library/yql/public/udf/service/exception_policy + ydb/library/yql/utils/backtrace + ydb/library/yql/utils/log + ydb/library/yql/providers/common/proto + ydb/library/yql/providers/common/udf_resolve + ydb/library/yql/providers/solomon/gateway + ydb/library/yql/providers/solomon/provider + ydb/library/yql/core/url_preprocessing + ydb/library/yql/providers/yt/gateway/native + ydb/library/yql/providers/yt/lib/log + ydb/library/yql/providers/yt/lib/yt_download + ydb/library/yql/providers/yt/provider + + yt/yql/plugin +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/yt/yql/plugin/plugin.cpp b/yt/yql/plugin/plugin.cpp new file mode 100644 index 0000000000..5ebfffb796 --- /dev/null +++ b/yt/yql/plugin/plugin.cpp @@ -0,0 +1,18 @@ +#include "plugin.h" + +#include <iostream> + +namespace NYT::NYqlPlugin { + +//////////////////////////////////////////////////////////////////////////////// + +Y_WEAK std::unique_ptr<IYqlPlugin> CreateYqlPlugin(TYqlPluginOptions& /*options*/) noexcept +{ + std::cerr << "No YQL plugin implementation is available; link against either " + << "yt/yql/plugin/native or yt/yql/plugin/dynamic" << std::endl; + exit(1); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYqlPlugin diff --git a/yt/yql/plugin/plugin.h b/yt/yql/plugin/plugin.h new file mode 100644 index 0000000000..46a08ba5e7 --- /dev/null +++ b/yt/yql/plugin/plugin.h @@ -0,0 +1,65 @@ +#pragma once + +#include <util/generic/hash.h> +#include <util/generic/string.h> + +#include <library/cpp/logger/log.h> + +#include <library/cpp/yt/yson_string/string.h> + +#include <optional> + +namespace NYT::NYqlPlugin { + +using namespace NYson; + +//////////////////////////////////////////////////////////////////////////////// + +class TYqlPluginOptions +{ +public: + TString MRJobBinary = "./mrjob"; + TString UdfDirectory; + + //! Mapping cluster name -> proxy address. + THashMap<TString, TString> Clusters; + std::optional<TString> DefaultCluster; + + TYsonString OperationAttributes; + + TString YTTokenPath; + + THolder<TLogBackend> LogBackend; + + std::optional<TString> YqlPluginSharedLibrary; +}; + +struct TQueryResult +{ + std::optional<TString> YsonResult; + std::optional<TString> Plan; + std::optional<TString> Statistics; + std::optional<TString> TaskInfo; + + //! YSON representation of a YT error. + std::optional<TString> YsonError; +}; + +//! This interface encapsulates YT <-> YQL integration. +//! There are two major implementation: one of them is based +//! on YQL code and another wraps the pure C bridge interface, which +//! is implemented by a dynamic library. +struct IYqlPlugin +{ + virtual TQueryResult Run(TString impersonationUser, TString queryText, TYsonString settings) noexcept = 0; + + virtual ~IYqlPlugin() = default; +}; + +//////////////////////////////////////////////////////////////////////////////// + +Y_WEAK std::unique_ptr<IYqlPlugin> CreateYqlPlugin(TYqlPluginOptions& options) noexcept; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYqlPlugin diff --git a/yt/yql/plugin/ya.make b/yt/yql/plugin/ya.make new file mode 100644 index 0000000000..7046752106 --- /dev/null +++ b/yt/yql/plugin/ya.make @@ -0,0 +1,19 @@ +LIBRARY() + +SRCS( + plugin.cpp +) + +END() + +RECURSE( + bridge +) + +IF (NOT OPENSOURCE) + # We do not bring YQL with us into open source. + RECURSE( + dynamic + native + ) +ENDIF() |