summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorvvvv <[email protected]>2023-05-18 15:20:10 +0300
committervvvv <[email protected]>2023-05-18 15:20:10 +0300
commit76f1d9f25a25d001f50b32113a5cc8f488d9d20a (patch)
tree69771a1658f390dd5cd579afac79f031b5cea3a7
parent0a48ad527344bcfac46ddddd263aba22ae420619 (diff)
Initial version of arrow kernel registry facade over computation graph
-rw-r--r--ydb/library/yql/core/CMakeLists.darwin-x86_64.txt1
-rw-r--r--ydb/library/yql/core/CMakeLists.linux-aarch64.txt1
-rw-r--r--ydb/library/yql/core/CMakeLists.linux-x86_64.txt1
-rw-r--r--ydb/library/yql/core/CMakeLists.windows-x86_64.txt1
-rw-r--r--ydb/library/yql/core/arrow_kernels/CMakeLists.txt10
-rw-r--r--ydb/library/yql/core/arrow_kernels/registry/CMakeLists.darwin-x86_64.txt24
-rw-r--r--ydb/library/yql/core/arrow_kernels/registry/CMakeLists.linux-aarch64.txt25
-rw-r--r--ydb/library/yql/core/arrow_kernels/registry/CMakeLists.linux-x86_64.txt25
-rw-r--r--ydb/library/yql/core/arrow_kernels/registry/CMakeLists.txt17
-rw-r--r--ydb/library/yql/core/arrow_kernels/registry/CMakeLists.windows-x86_64.txt24
-rw-r--r--ydb/library/yql/core/arrow_kernels/registry/registry.cpp84
-rw-r--r--ydb/library/yql/core/arrow_kernels/registry/registry.h14
-rw-r--r--ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.darwin-x86_64.txt76
-rw-r--r--ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.linux-aarch64.txt79
-rw-r--r--ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.linux-x86_64.txt81
-rw-r--r--ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.txt17
-rw-r--r--ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.windows-x86_64.txt69
-rw-r--r--ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp84
-rw-r--r--ydb/library/yql/core/arrow_kernels/request/CMakeLists.darwin-x86_64.txt23
-rw-r--r--ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-aarch64.txt24
-rw-r--r--ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-x86_64.txt24
-rw-r--r--ydb/library/yql/core/arrow_kernels/request/CMakeLists.txt17
-rw-r--r--ydb/library/yql/core/arrow_kernels/request/CMakeLists.windows-x86_64.txt23
-rw-r--r--ydb/library/yql/core/arrow_kernels/request/request.cpp82
-rw-r--r--ydb/library/yql/core/arrow_kernels/request/request.h41
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_block_logical.cpp66
26 files changed, 925 insertions, 8 deletions
diff --git a/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt
index 309e84a6e4b..08c0d7bbdb4 100644
--- a/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(arrow_kernels)
add_subdirectory(common_opt)
add_subdirectory(credentials)
add_subdirectory(expr_nodes)
diff --git a/ydb/library/yql/core/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/CMakeLists.linux-aarch64.txt
index 84c9994fd8d..f0ee199095c 100644
--- a/ydb/library/yql/core/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/core/CMakeLists.linux-aarch64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(arrow_kernels)
add_subdirectory(common_opt)
add_subdirectory(credentials)
add_subdirectory(expr_nodes)
diff --git a/ydb/library/yql/core/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/CMakeLists.linux-x86_64.txt
index 84c9994fd8d..f0ee199095c 100644
--- a/ydb/library/yql/core/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/core/CMakeLists.linux-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(arrow_kernels)
add_subdirectory(common_opt)
add_subdirectory(credentials)
add_subdirectory(expr_nodes)
diff --git a/ydb/library/yql/core/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/CMakeLists.windows-x86_64.txt
index 309e84a6e4b..08c0d7bbdb4 100644
--- a/ydb/library/yql/core/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/core/CMakeLists.windows-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(arrow_kernels)
add_subdirectory(common_opt)
add_subdirectory(credentials)
add_subdirectory(expr_nodes)
diff --git a/ydb/library/yql/core/arrow_kernels/CMakeLists.txt b/ydb/library/yql/core/arrow_kernels/CMakeLists.txt
new file mode 100644
index 00000000000..373fadacb92
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/CMakeLists.txt
@@ -0,0 +1,10 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(registry)
+add_subdirectory(request)
diff --git a/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..de0155c3cb3
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,24 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(ut)
+
+add_library(core-arrow_kernels-registry)
+target_compile_options(core-arrow_kernels-registry PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(core-arrow_kernels-registry PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ libs-apache-arrow
+ library-yql-minikql
+ yql-minikql-computation
+)
+target_sources(core-arrow_kernels-registry PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry/registry.cpp
+)
diff --git a/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..12537659d56
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,25 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(ut)
+
+add_library(core-arrow_kernels-registry)
+target_compile_options(core-arrow_kernels-registry PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(core-arrow_kernels-registry PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ libs-apache-arrow
+ library-yql-minikql
+ yql-minikql-computation
+)
+target_sources(core-arrow_kernels-registry PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry/registry.cpp
+)
diff --git a/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..12537659d56
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,25 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(ut)
+
+add_library(core-arrow_kernels-registry)
+target_compile_options(core-arrow_kernels-registry PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(core-arrow_kernels-registry PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ libs-apache-arrow
+ library-yql-minikql
+ yql-minikql-computation
+)
+target_sources(core-arrow_kernels-registry PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry/registry.cpp
+)
diff --git a/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.txt b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..de0155c3cb3
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,24 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(ut)
+
+add_library(core-arrow_kernels-registry)
+target_compile_options(core-arrow_kernels-registry PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(core-arrow_kernels-registry PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ libs-apache-arrow
+ library-yql-minikql
+ yql-minikql-computation
+)
+target_sources(core-arrow_kernels-registry PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry/registry.cpp
+)
diff --git a/ydb/library/yql/core/arrow_kernels/registry/registry.cpp b/ydb/library/yql/core/arrow_kernels/registry/registry.cpp
new file mode 100644
index 00000000000..f1940ffabbf
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/registry/registry.cpp
@@ -0,0 +1,84 @@
+#include "registry.h"
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_node_serialization.h>
+
+#include <memory>
+
+namespace NYql {
+
+namespace {
+ class TLoader : std::enable_shared_from_this<TLoader> {
+ public:
+ TLoader()
+ : Alloc_(__LOCATION__)
+ , Env_(Alloc_)
+ {
+ Alloc_.Release();
+ }
+
+ void Init(const TString& serialized,
+ const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry,
+ const NKikimr::NMiniKQL::TComputationNodeFactory& nodeFactory) {
+ TGuard<NKikimr::NMiniKQL::TScopedAlloc> allocGuard(Alloc_);
+ Pgm_ = NKikimr::NMiniKQL::DeserializeRuntimeNode(serialized, Env_);
+ Explorer_.Walk(Pgm_.GetNode(), Env_);
+ NKikimr::NMiniKQL::TComputationPatternOpts opts(Alloc_.Ref(), Env_, nodeFactory,
+ &functionRegistry, NUdf::EValidateMode::None, NUdf::EValidatePolicy::Exception, "OFF", NKikimr::NMiniKQL::EGraphPerProcess::Multi);
+ std::vector<NKikimr::NMiniKQL::TNode*> entryPoints;
+ for (const auto& node : Explorer_.GetNodes()) {
+ if (node->GetType()->IsCallable() && AS_TYPE(NKikimr::NMiniKQL::TCallableType, node->GetType())->GetName() == "Arg") {
+ entryPoints.emplace_back(node);
+ }
+ }
+
+ Pattern_ = NKikimr::NMiniKQL::MakeComputationPattern(Explorer_, Pgm_, entryPoints, opts);
+ RandomProvider_ = CreateDefaultRandomProvider();
+ TimeProvider_ = CreateDefaultTimeProvider();
+
+ Graph_ = Pattern_->Clone(opts.ToComputationOptions(*RandomProvider_, *TimeProvider_));
+ NKikimr::NMiniKQL::TBindTerminator terminator(Graph_->GetTerminator());
+ Topology_ = Graph_->GetKernelsTopology();
+ MKQL_ENSURE(Topology_->Items.size() >= 1, "Expected at least one kernel");
+ }
+
+ ~TLoader() {
+ Alloc_.Acquire();
+ }
+
+ ui32 GetKernelsCount() const {
+ return Topology_->Items.size() - 1;
+ }
+
+ const arrow::compute::ScalarKernel* GetKernel(ui32 index) const {
+ MKQL_ENSURE(index < Topology_->Items.size() - 1, "Bad kernel index");
+ return &Topology_->Items[index].Node->GetArrowKernel();
+ }
+
+ private:
+ NKikimr::NMiniKQL::TScopedAlloc Alloc_;
+ NKikimr::NMiniKQL::TTypeEnvironment Env_;
+ NKikimr::NMiniKQL::TRuntimeNode Pgm_;
+ NKikimr::NMiniKQL::TExploringNodeVisitor Explorer_;
+ NKikimr::NMiniKQL::IComputationPattern::TPtr Pattern_;
+ TIntrusivePtr<IRandomProvider> RandomProvider_;
+ TIntrusivePtr<ITimeProvider> TimeProvider_;
+ THolder<NKikimr::NMiniKQL::IComputationGraph> Graph_;
+ const NKikimr::NMiniKQL::TArrowKernelsTopology* Topology_;
+ };
+}
+
+std::vector<std::shared_ptr<const arrow::compute::ScalarKernel>> LoadKernels(const TString& serialized,
+ const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry,
+ const NKikimr::NMiniKQL::TComputationNodeFactory& nodeFactory) {
+ auto loader = std::make_shared<TLoader>();
+ loader->Init(serialized, functionRegistry, nodeFactory);
+ std::vector<std::shared_ptr<const arrow::compute::ScalarKernel>> ret(loader->GetKernelsCount());
+ auto deleter = [loader](const arrow::compute::ScalarKernel*) {};
+ for (ui32 i = 0; i < ret.size(); ++i) {
+ ret[i] = std::shared_ptr<const arrow::compute::ScalarKernel>(loader->GetKernel(i), deleter);
+ }
+
+ return ret;
+}
+
+} \ No newline at end of file
diff --git a/ydb/library/yql/core/arrow_kernels/registry/registry.h b/ydb/library/yql/core/arrow_kernels/registry/registry.h
new file mode 100644
index 00000000000..7bff5f8c9ca
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/registry/registry.h
@@ -0,0 +1,14 @@
+#pragma once
+#include <arrow/compute/kernel.h>
+#include <memory>
+#include <vector>
+#include <ydb/library/yql/minikql/mkql_function_registry.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node.h>
+
+namespace NYql {
+
+std::vector<std::shared_ptr<const arrow::compute::ScalarKernel>> LoadKernels(const TString& serialized,
+ const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry,
+ const NKikimr::NMiniKQL::TComputationNodeFactory& nodeFactory);
+
+} \ No newline at end of file
diff --git a/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..ae4c3e50f47
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,76 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-core-arrow_kernels-registry-ut)
+target_compile_options(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry
+)
+target_link_libraries(ydb-library-yql-core-arrow_kernels-registry-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ core-arrow_kernels-registry
+ core-arrow_kernels-request
+ yql-public-udf
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+ yql-minikql-invoke_builtins
+ yql-minikql-comp_nodes
+)
+target_link_options(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ TEST_TARGET
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-core-arrow_kernels-registry-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-core-arrow_kernels-registry-ut)
diff --git a/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..6bf2afe1c24
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,79 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-core-arrow_kernels-registry-ut)
+target_compile_options(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry
+)
+target_link_libraries(ydb-library-yql-core-arrow_kernels-registry-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-testing-unittest_main
+ core-arrow_kernels-registry
+ core-arrow_kernels-request
+ yql-public-udf
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+ yql-minikql-invoke_builtins
+ yql-minikql-comp_nodes
+)
+target_link_options(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ TEST_TARGET
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-core-arrow_kernels-registry-ut
+ cpp-malloc-jemalloc
+)
+vcs_info(ydb-library-yql-core-arrow_kernels-registry-ut)
diff --git a/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..f1a3e5291d3
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,81 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-core-arrow_kernels-registry-ut)
+target_compile_options(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry
+)
+target_link_libraries(ydb-library-yql-core-arrow_kernels-registry-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ core-arrow_kernels-registry
+ core-arrow_kernels-request
+ yql-public-udf
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+ yql-minikql-invoke_builtins
+ yql-minikql-comp_nodes
+)
+target_link_options(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ TEST_TARGET
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-core-arrow_kernels-registry-ut
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(ydb-library-yql-core-arrow_kernels-registry-ut)
diff --git a/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.txt b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..55f8a282d87
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,69 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-core-arrow_kernels-registry-ut)
+target_compile_options(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry
+)
+target_link_libraries(ydb-library-yql-core-arrow_kernels-registry-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ core-arrow_kernels-registry
+ core-arrow_kernels-request
+ yql-public-udf
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+ yql-minikql-invoke_builtins
+ yql-minikql-comp_nodes
+)
+target_sources(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ TEST_TARGET
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-arrow_kernels-registry-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-core-arrow_kernels-registry-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-core-arrow_kernels-registry-ut)
diff --git a/ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp b/ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp
new file mode 100644
index 00000000000..1acaa8637ba
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp
@@ -0,0 +1,84 @@
+#include <ydb/library/yql/core/arrow_kernels/request/request.h>
+#include <ydb/library/yql/core/arrow_kernels/registry/registry.h>
+#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
+#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+using namespace NYql;
+using namespace NKikimr::NMiniKQL;
+
+template <typename F>
+void TestOne(F&& f) {
+ TExprContext ctx;
+ auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
+ auto nodeFactory = GetBuiltinFactory();
+ TKernelRequestBuilder b(*functionRegistry);
+ auto index = f(b, ctx);
+ UNIT_ASSERT_VALUES_EQUAL(index, 0);
+ auto s = b.Serialize();
+ auto v = LoadKernels(s, *functionRegistry, nodeFactory);
+ UNIT_ASSERT_VALUES_EQUAL(v.size(), 1);
+}
+
+Y_UNIT_TEST_SUITE(TKernelRegistryTest) {
+ Y_UNIT_TEST(TestZeroKernels) {
+ auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
+ auto nodeFactory = GetBuiltinFactory();
+ TKernelRequestBuilder b(*functionRegistry);
+ auto s = b.Serialize();
+ auto v = LoadKernels(s, *functionRegistry, nodeFactory);
+ UNIT_ASSERT_VALUES_EQUAL(v.size(), 0);
+ }
+
+ Y_UNIT_TEST(TestTwoKernels) {
+ TExprContext ctx;
+ auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
+ auto nodeFactory = GetBuiltinFactory();
+ TKernelRequestBuilder b(*functionRegistry);
+ auto blockInt32Type = ctx.template MakeType<TBlockExprType>(ctx.template MakeType<TDataExprType>(EDataSlot::Int32));
+ auto index1 = b.AddBinaryOp(TKernelRequestBuilder::EBinaryOp::Add, blockInt32Type, blockInt32Type, blockInt32Type);
+ UNIT_ASSERT_VALUES_EQUAL(index1, 0);
+ auto index2 = b.AddBinaryOp(TKernelRequestBuilder::EBinaryOp::Sub, blockInt32Type, blockInt32Type, blockInt32Type);
+ UNIT_ASSERT_VALUES_EQUAL(index2, 1);
+ auto s = b.Serialize();
+ auto v = LoadKernels(s, *functionRegistry, nodeFactory);
+ UNIT_ASSERT_VALUES_EQUAL(v.size(), 2);
+ }
+
+ Y_UNIT_TEST(TestNot) {
+ TestOne([](auto& b,auto& ctx) {
+ auto blockBoolType = ctx.template MakeType<TBlockExprType>(ctx.template MakeType<TDataExprType>(EDataSlot::Bool));
+ return b.AddUnaryOp(TKernelRequestBuilder::EUnaryOp::Not, blockBoolType, blockBoolType);
+ });
+ }
+
+ Y_UNIT_TEST(TestAdd) {
+ TestOne([](auto& b,auto& ctx) {
+ auto blockInt32Type = ctx.template MakeType<TBlockExprType>(ctx.template MakeType<TDataExprType>(EDataSlot::Int32));
+ return b.AddBinaryOp(TKernelRequestBuilder::EBinaryOp::Add, blockInt32Type, blockInt32Type, blockInt32Type);
+ });
+ }
+
+ Y_UNIT_TEST(TestSub) {
+ TestOne([](auto& b,auto& ctx) {
+ auto blockInt32Type = ctx.template MakeType<TBlockExprType>(ctx.template MakeType<TDataExprType>(EDataSlot::Int32));
+ return b.AddBinaryOp(TKernelRequestBuilder::EBinaryOp::Sub, blockInt32Type, blockInt32Type, blockInt32Type);
+ });
+ }
+
+ Y_UNIT_TEST(TestMul) {
+ TestOne([](auto& b,auto& ctx) {
+ auto blockInt32Type = ctx.template MakeType<TBlockExprType>(ctx.template MakeType<TDataExprType>(EDataSlot::Int32));
+ return b.AddBinaryOp(TKernelRequestBuilder::EBinaryOp::Mul, blockInt32Type, blockInt32Type, blockInt32Type);
+ });
+ }
+
+ Y_UNIT_TEST(TestDiv) {
+ TestOne([](auto& b,auto& ctx) {
+ auto blockInt32Type = ctx.template MakeType<TBlockExprType>(ctx.template MakeType<TDataExprType>(EDataSlot::Int32));
+ auto blockOptInt32Type = ctx.template MakeType<TBlockExprType>(ctx.template MakeType<TOptionalExprType>(ctx.template MakeType<TDataExprType>(EDataSlot::Int32)));
+ return b.AddBinaryOp(TKernelRequestBuilder::EBinaryOp::Div, blockInt32Type, blockInt32Type, blockOptInt32Type);
+ });
+ }
+}
diff --git a/ydb/library/yql/core/arrow_kernels/request/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..5877718c2a4
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,23 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-arrow_kernels-request)
+target_compile_options(core-arrow_kernels-request PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(core-arrow_kernels-request PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-yql-ast
+ library-yql-minikql
+ providers-common-mkql
+)
+target_sources(core-arrow_kernels-request PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/request/request.cpp
+)
diff --git a/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..a38846dc53d
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,24 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-arrow_kernels-request)
+target_compile_options(core-arrow_kernels-request PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(core-arrow_kernels-request PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-yql-ast
+ library-yql-minikql
+ providers-common-mkql
+)
+target_sources(core-arrow_kernels-request PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/request/request.cpp
+)
diff --git a/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..a38846dc53d
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,24 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-arrow_kernels-request)
+target_compile_options(core-arrow_kernels-request PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(core-arrow_kernels-request PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-yql-ast
+ library-yql-minikql
+ providers-common-mkql
+)
+target_sources(core-arrow_kernels-request PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/request/request.cpp
+)
diff --git a/ydb/library/yql/core/arrow_kernels/request/CMakeLists.txt b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/core/arrow_kernels/request/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..5877718c2a4
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,23 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(core-arrow_kernels-request)
+target_compile_options(core-arrow_kernels-request PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(core-arrow_kernels-request PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-yql-ast
+ library-yql-minikql
+ providers-common-mkql
+)
+target_sources(core-arrow_kernels-request PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/request/request.cpp
+)
diff --git a/ydb/library/yql/core/arrow_kernels/request/request.cpp b/ydb/library/yql/core/arrow_kernels/request/request.cpp
new file mode 100644
index 00000000000..8450acbce97
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/request/request.cpp
@@ -0,0 +1,82 @@
+#include "request.h"
+#include <ydb/library/yql/providers/common/mkql/yql_type_mkql.h>
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_node_serialization.h>
+
+namespace NYql {
+
+TKernelRequestBuilder::TKernelRequestBuilder(const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry)
+ : Alloc_(__LOCATION__)
+ , Env_(Alloc_)
+ , Pb_(Env_, functionRegistry)
+{
+ Alloc_.Release();
+}
+
+TKernelRequestBuilder::~TKernelRequestBuilder() {
+ Alloc_.Acquire();
+}
+
+ui32 TKernelRequestBuilder::AddUnaryOp(EUnaryOp op, const TTypeAnnotationNode* arg1Type, const TTypeAnnotationNode* retType) {
+ TGuard<NKikimr::NMiniKQL::TScopedAlloc> allocGuard(Alloc_);
+ auto returnType = MakeType(retType);
+ Y_UNUSED(returnType);
+ auto arg1 = MakeArg(arg1Type);
+ switch (op) {
+ case EUnaryOp::Not:
+ Items_.emplace_back(Pb_.BlockNot(arg1));
+ break;
+ }
+
+ return Items_.size() - 1;
+}
+
+ui32 TKernelRequestBuilder::AddBinaryOp(EBinaryOp op, const TTypeAnnotationNode* arg1Type, const TTypeAnnotationNode* arg2Type, const TTypeAnnotationNode* retType) {
+ TGuard<NKikimr::NMiniKQL::TScopedAlloc> allocGuard(Alloc_);
+ auto returnType = MakeType(retType);
+ auto arg1 = MakeArg(arg1Type);
+ auto arg2 = MakeArg(arg2Type);
+ switch (op) {
+ case EBinaryOp::Add:
+ Items_.emplace_back(Pb_.BlockFunc("Add", returnType, { arg1, arg2 }));
+ break;
+ case EBinaryOp::Sub:
+ Items_.emplace_back(Pb_.BlockFunc("Sub", returnType, { arg1, arg2 }));
+ break;
+ case EBinaryOp::Mul:
+ Items_.emplace_back(Pb_.BlockFunc("Mul", returnType, { arg1, arg2 }));
+ break;
+ case EBinaryOp::Div:
+ Items_.emplace_back(Pb_.BlockFunc("Div", returnType, { arg1, arg2 }));
+ break;
+ }
+
+ return Items_.size() - 1;
+}
+
+TString TKernelRequestBuilder::Serialize() {
+ TGuard<NKikimr::NMiniKQL::TScopedAlloc> allocGuard(Alloc_);
+ auto tuple = Items_.empty() ? Pb_.AsScalar(Pb_.NewEmptyTuple()) : Pb_.BlockAsTuple(Items_);
+ return NKikimr::NMiniKQL::SerializeRuntimeNode(tuple, Env_);
+}
+
+NKikimr::NMiniKQL::TRuntimeNode TKernelRequestBuilder::MakeArg(const TTypeAnnotationNode* type) {
+ auto [it, inserted] = CachedArgs_.emplace(type, NKikimr::NMiniKQL::TRuntimeNode());
+ if (!inserted) {
+ return it->second;
+ }
+
+ return it->second = Pb_.Arg(MakeType(type));
+}
+
+NKikimr::NMiniKQL::TBlockType* TKernelRequestBuilder::MakeType(const TTypeAnnotationNode* type) {
+ TStringStream err;
+ auto ret = NCommon::BuildType(*type, Pb_, err);
+ if (!ret) {
+ ythrow yexception() << err.Str();
+ }
+
+ return AS_TYPE(NKikimr::NMiniKQL::TBlockType, ret);
+}
+
+}
diff --git a/ydb/library/yql/core/arrow_kernels/request/request.h b/ydb/library/yql/core/arrow_kernels/request/request.h
new file mode 100644
index 00000000000..f3eb860a0d4
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/request/request.h
@@ -0,0 +1,41 @@
+#pragma once
+#include <ydb/library/yql/ast/yql_expr.h>
+#include <ydb/library/yql/minikql/mkql_program_builder.h>
+
+#include <unordered_map>
+
+namespace NYql {
+
+class TKernelRequestBuilder {
+public:
+ enum EUnaryOp {
+ Not
+ };
+
+ enum EBinaryOp {
+ Add,
+ Sub,
+ Mul,
+ Div
+ };
+
+ TKernelRequestBuilder(const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry);
+ ~TKernelRequestBuilder();
+
+ ui32 AddUnaryOp(EUnaryOp op, const TTypeAnnotationNode* arg1Type, const TTypeAnnotationNode* retType);
+ ui32 AddBinaryOp(EBinaryOp op, const TTypeAnnotationNode* arg1Type, const TTypeAnnotationNode* arg2Type, const TTypeAnnotationNode* retType);
+ TString Serialize();
+
+private:
+ NKikimr::NMiniKQL::TRuntimeNode MakeArg(const TTypeAnnotationNode* type);
+ NKikimr::NMiniKQL::TBlockType* MakeType(const TTypeAnnotationNode* type);
+private:
+ NKikimr::NMiniKQL::TScopedAlloc Alloc_;
+ NKikimr::NMiniKQL::TTypeEnvironment Env_;
+ NKikimr::NMiniKQL::TProgramBuilder Pb_;
+ TVector<NKikimr::NMiniKQL::TRuntimeNode> Items_;
+ std::unordered_map<const TTypeAnnotationNode*, NKikimr::NMiniKQL::TRuntimeNode> CachedArgs_;
+};
+
+
+} \ No newline at end of file
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_block_logical.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_block_logical.cpp
index c33168c4108..e2cbcc9e845 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_block_logical.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_block_logical.cpp
@@ -493,6 +493,49 @@ private:
};
class TBlockNotWrapper : public TMutableComputationNode<TBlockNotWrapper> {
+friend class TArrowNode;
+public:
+ class TArrowNode : public IArrowKernelComputationNode {
+ public:
+ TArrowNode(const TBlockNotWrapper* parent)
+ : Parent_(parent)
+ , ArgsValuesDescr_({arrow::uint8()})
+ , Kernel_({arrow::uint8()}, arrow::uint8(), [parent](arrow::compute::KernelContext* ctx, const arrow::compute::ExecBatch& batch, arrow::Datum* res) {
+ *res = parent->CalculateImpl(MakeDatumProvider(batch.values[0]), *ctx->memory_pool());
+ return arrow::Status::OK();
+ })
+ {
+ Kernel_.null_handling = arrow::compute::NullHandling::COMPUTED_NO_PREALLOCATE;
+ Kernel_.mem_allocation = arrow::compute::MemAllocation::NO_PREALLOCATE;
+ }
+
+ TStringBuf GetKernelName() const final {
+ return "Not";
+ }
+
+ const arrow::compute::ScalarKernel& GetArrowKernel() const {
+ return Kernel_;
+ }
+
+ const std::vector<arrow::ValueDescr>& GetArgsDesc() const {
+ return ArgsValuesDescr_;
+ }
+
+ const IComputationNode* GetArgument(ui32 index) const {
+ switch (index) {
+ case 0:
+ return Parent_->Value;
+ default:
+ throw yexception() << "Bad argument index";
+ }
+ }
+
+ private:
+ const TBlockNotWrapper* Parent_;
+ const std::vector<arrow::ValueDescr> ArgsValuesDescr_;
+ arrow::compute::ScalarKernel Kernel_;
+ };
+
public:
TBlockNotWrapper(TComputationMutables& mutables, IComputationNode* value)
: TMutableComputationNode(mutables)
@@ -500,25 +543,32 @@ public:
{
}
- NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const {
- auto value = Value->GetValue(ctx);
- const auto& datum = TArrowBlock::From(value).GetDatum();
+ std::unique_ptr<IArrowKernelComputationNode> PrepareArrowKernelComputationNode(TComputationContext& ctx) const final {
+ Y_UNUSED(ctx);
+ return std::make_unique<TArrowNode>(this);
+ }
+ arrow::Datum CalculateImpl(const TDatumProvider& valueProv, arrow::MemoryPool& memoryPool) const {
+ auto datum = valueProv();
if (datum.null_count() == datum.length()) {
- return value.Release();
+ return datum;
}
if (datum.is_scalar()) {
Y_VERIFY(datum.scalar()->is_valid);
ui8 negated = (~datum.scalar_as<arrow::UInt8Scalar>().value) & 1u;
- return ctx.HolderFactory.CreateArrowBlock(arrow::Datum(negated));
+ return arrow::Datum(negated);
}
auto arr = datum.array();
- std::shared_ptr<arrow::Buffer> bitmap = CopyBitmap(&ctx.ArrowMemoryPool, arr->buffers[0], arr->offset, arr->length);
- std::shared_ptr<arrow::Buffer> data = arrow::AllocateBuffer(arr->length, &ctx.ArrowMemoryPool).ValueOrDie();
+ std::shared_ptr<arrow::Buffer> bitmap = CopyBitmap(&memoryPool, arr->buffers[0], arr->offset, arr->length);
+ std::shared_ptr<arrow::Buffer> data = arrow::AllocateBuffer(arr->length, &memoryPool).ValueOrDie();
NegateSparseBitmap(data->mutable_data(), arr->GetValues<ui8>(1), arr->length);
- return ctx.HolderFactory.CreateArrowBlock(arrow::ArrayData::Make(arr->type, arr->length, { bitmap, data }));
+ return arrow::ArrayData::Make(arr->type, arr->length, { bitmap, data });
+ }
+
+ NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const {
+ return ctx.HolderFactory.CreateArrowBlock(CalculateImpl(MakeDatumProvider(Value, ctx), ctx.ArrowMemoryPool));
}
private: