diff options
author | vvvv <[email protected]> | 2023-05-18 15:20:10 +0300 |
---|---|---|
committer | vvvv <[email protected]> | 2023-05-18 15:20:10 +0300 |
commit | 76f1d9f25a25d001f50b32113a5cc8f488d9d20a (patch) | |
tree | 69771a1658f390dd5cd579afac79f031b5cea3a7 | |
parent | 0a48ad527344bcfac46ddddd263aba22ae420619 (diff) |
Initial version of arrow kernel registry facade over computation graph
26 files changed, 925 insertions, 8 deletions
diff --git a/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt index 309e84a6e4b..08c0d7bbdb4 100644 --- a/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(arrow_kernels) add_subdirectory(common_opt) add_subdirectory(credentials) add_subdirectory(expr_nodes) diff --git a/ydb/library/yql/core/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/CMakeLists.linux-aarch64.txt index 84c9994fd8d..f0ee199095c 100644 --- a/ydb/library/yql/core/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/core/CMakeLists.linux-aarch64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(arrow_kernels) add_subdirectory(common_opt) add_subdirectory(credentials) add_subdirectory(expr_nodes) diff --git a/ydb/library/yql/core/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/CMakeLists.linux-x86_64.txt index 84c9994fd8d..f0ee199095c 100644 --- a/ydb/library/yql/core/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/core/CMakeLists.linux-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(arrow_kernels) add_subdirectory(common_opt) add_subdirectory(credentials) add_subdirectory(expr_nodes) diff --git a/ydb/library/yql/core/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/CMakeLists.windows-x86_64.txt index 309e84a6e4b..08c0d7bbdb4 100644 --- a/ydb/library/yql/core/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/core/CMakeLists.windows-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(arrow_kernels) add_subdirectory(common_opt) add_subdirectory(credentials) add_subdirectory(expr_nodes) diff --git a/ydb/library/yql/core/arrow_kernels/CMakeLists.txt b/ydb/library/yql/core/arrow_kernels/CMakeLists.txt new file mode 100644 index 00000000000..373fadacb92 --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/CMakeLists.txt @@ -0,0 +1,10 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(registry) +add_subdirectory(request) diff --git a/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..de0155c3cb3 --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,24 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(ut) + +add_library(core-arrow_kernels-registry) +target_compile_options(core-arrow_kernels-registry PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(core-arrow_kernels-registry PUBLIC + contrib-libs-cxxsupp + yutil + libs-apache-arrow + library-yql-minikql + yql-minikql-computation +) +target_sources(core-arrow_kernels-registry PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry/registry.cpp +) diff --git a/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..12537659d56 --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.linux-aarch64.txt @@ -0,0 +1,25 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(ut) + +add_library(core-arrow_kernels-registry) +target_compile_options(core-arrow_kernels-registry PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(core-arrow_kernels-registry PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + libs-apache-arrow + library-yql-minikql + yql-minikql-computation +) +target_sources(core-arrow_kernels-registry PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry/registry.cpp +) diff --git a/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..12537659d56 --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.linux-x86_64.txt @@ -0,0 +1,25 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(ut) + +add_library(core-arrow_kernels-registry) +target_compile_options(core-arrow_kernels-registry PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(core-arrow_kernels-registry PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + libs-apache-arrow + library-yql-minikql + yql-minikql-computation +) +target_sources(core-arrow_kernels-registry PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry/registry.cpp +) diff --git a/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.txt b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..de0155c3cb3 --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/registry/CMakeLists.windows-x86_64.txt @@ -0,0 +1,24 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(ut) + +add_library(core-arrow_kernels-registry) +target_compile_options(core-arrow_kernels-registry PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(core-arrow_kernels-registry PUBLIC + contrib-libs-cxxsupp + yutil + libs-apache-arrow + library-yql-minikql + yql-minikql-computation +) +target_sources(core-arrow_kernels-registry PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry/registry.cpp +) diff --git a/ydb/library/yql/core/arrow_kernels/registry/registry.cpp b/ydb/library/yql/core/arrow_kernels/registry/registry.cpp new file mode 100644 index 00000000000..f1940ffabbf --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/registry/registry.cpp @@ -0,0 +1,84 @@ +#include "registry.h" +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_node_serialization.h> + +#include <memory> + +namespace NYql { + +namespace { + class TLoader : std::enable_shared_from_this<TLoader> { + public: + TLoader() + : Alloc_(__LOCATION__) + , Env_(Alloc_) + { + Alloc_.Release(); + } + + void Init(const TString& serialized, + const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry, + const NKikimr::NMiniKQL::TComputationNodeFactory& nodeFactory) { + TGuard<NKikimr::NMiniKQL::TScopedAlloc> allocGuard(Alloc_); + Pgm_ = NKikimr::NMiniKQL::DeserializeRuntimeNode(serialized, Env_); + Explorer_.Walk(Pgm_.GetNode(), Env_); + NKikimr::NMiniKQL::TComputationPatternOpts opts(Alloc_.Ref(), Env_, nodeFactory, + &functionRegistry, NUdf::EValidateMode::None, NUdf::EValidatePolicy::Exception, "OFF", NKikimr::NMiniKQL::EGraphPerProcess::Multi); + std::vector<NKikimr::NMiniKQL::TNode*> entryPoints; + for (const auto& node : Explorer_.GetNodes()) { + if (node->GetType()->IsCallable() && AS_TYPE(NKikimr::NMiniKQL::TCallableType, node->GetType())->GetName() == "Arg") { + entryPoints.emplace_back(node); + } + } + + Pattern_ = NKikimr::NMiniKQL::MakeComputationPattern(Explorer_, Pgm_, entryPoints, opts); + RandomProvider_ = CreateDefaultRandomProvider(); + TimeProvider_ = CreateDefaultTimeProvider(); + + Graph_ = Pattern_->Clone(opts.ToComputationOptions(*RandomProvider_, *TimeProvider_)); + NKikimr::NMiniKQL::TBindTerminator terminator(Graph_->GetTerminator()); + Topology_ = Graph_->GetKernelsTopology(); + MKQL_ENSURE(Topology_->Items.size() >= 1, "Expected at least one kernel"); + } + + ~TLoader() { + Alloc_.Acquire(); + } + + ui32 GetKernelsCount() const { + return Topology_->Items.size() - 1; + } + + const arrow::compute::ScalarKernel* GetKernel(ui32 index) const { + MKQL_ENSURE(index < Topology_->Items.size() - 1, "Bad kernel index"); + return &Topology_->Items[index].Node->GetArrowKernel(); + } + + private: + NKikimr::NMiniKQL::TScopedAlloc Alloc_; + NKikimr::NMiniKQL::TTypeEnvironment Env_; + NKikimr::NMiniKQL::TRuntimeNode Pgm_; + NKikimr::NMiniKQL::TExploringNodeVisitor Explorer_; + NKikimr::NMiniKQL::IComputationPattern::TPtr Pattern_; + TIntrusivePtr<IRandomProvider> RandomProvider_; + TIntrusivePtr<ITimeProvider> TimeProvider_; + THolder<NKikimr::NMiniKQL::IComputationGraph> Graph_; + const NKikimr::NMiniKQL::TArrowKernelsTopology* Topology_; + }; +} + +std::vector<std::shared_ptr<const arrow::compute::ScalarKernel>> LoadKernels(const TString& serialized, + const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry, + const NKikimr::NMiniKQL::TComputationNodeFactory& nodeFactory) { + auto loader = std::make_shared<TLoader>(); + loader->Init(serialized, functionRegistry, nodeFactory); + std::vector<std::shared_ptr<const arrow::compute::ScalarKernel>> ret(loader->GetKernelsCount()); + auto deleter = [loader](const arrow::compute::ScalarKernel*) {}; + for (ui32 i = 0; i < ret.size(); ++i) { + ret[i] = std::shared_ptr<const arrow::compute::ScalarKernel>(loader->GetKernel(i), deleter); + } + + return ret; +} + +}
\ No newline at end of file diff --git a/ydb/library/yql/core/arrow_kernels/registry/registry.h b/ydb/library/yql/core/arrow_kernels/registry/registry.h new file mode 100644 index 00000000000..7bff5f8c9ca --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/registry/registry.h @@ -0,0 +1,14 @@ +#pragma once +#include <arrow/compute/kernel.h> +#include <memory> +#include <vector> +#include <ydb/library/yql/minikql/mkql_function_registry.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node.h> + +namespace NYql { + +std::vector<std::shared_ptr<const arrow::compute::ScalarKernel>> LoadKernels(const TString& serialized, + const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry, + const NKikimr::NMiniKQL::TComputationNodeFactory& nodeFactory); + +}
\ No newline at end of file diff --git a/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..ae4c3e50f47 --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,76 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-core-arrow_kernels-registry-ut) +target_compile_options(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry +) +target_link_libraries(ydb-library-yql-core-arrow_kernels-registry-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + core-arrow_kernels-registry + core-arrow_kernels-request + yql-public-udf + udf-service-exception_policy + yql-sql-pg_dummy + yql-minikql-invoke_builtins + yql-minikql-comp_nodes +) +target_link_options(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp +) +set_property( + TARGET + ydb-library-yql-core-arrow_kernels-registry-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-core-arrow_kernels-registry-ut + TEST_TARGET + ydb-library-yql-core-arrow_kernels-registry-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-core-arrow_kernels-registry-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-core-arrow_kernels-registry-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-core-arrow_kernels-registry-ut + system_allocator +) +vcs_info(ydb-library-yql-core-arrow_kernels-registry-ut) diff --git a/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..6bf2afe1c24 --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.linux-aarch64.txt @@ -0,0 +1,79 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-core-arrow_kernels-registry-ut) +target_compile_options(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry +) +target_link_libraries(ydb-library-yql-core-arrow_kernels-registry-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + core-arrow_kernels-registry + core-arrow_kernels-request + yql-public-udf + udf-service-exception_policy + yql-sql-pg_dummy + yql-minikql-invoke_builtins + yql-minikql-comp_nodes +) +target_link_options(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp +) +set_property( + TARGET + ydb-library-yql-core-arrow_kernels-registry-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-core-arrow_kernels-registry-ut + TEST_TARGET + ydb-library-yql-core-arrow_kernels-registry-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-core-arrow_kernels-registry-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-core-arrow_kernels-registry-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-core-arrow_kernels-registry-ut + cpp-malloc-jemalloc +) +vcs_info(ydb-library-yql-core-arrow_kernels-registry-ut) diff --git a/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..f1a3e5291d3 --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.linux-x86_64.txt @@ -0,0 +1,81 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-core-arrow_kernels-registry-ut) +target_compile_options(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry +) +target_link_libraries(ydb-library-yql-core-arrow_kernels-registry-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + core-arrow_kernels-registry + core-arrow_kernels-request + yql-public-udf + udf-service-exception_policy + yql-sql-pg_dummy + yql-minikql-invoke_builtins + yql-minikql-comp_nodes +) +target_link_options(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp +) +set_property( + TARGET + ydb-library-yql-core-arrow_kernels-registry-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-core-arrow_kernels-registry-ut + TEST_TARGET + ydb-library-yql-core-arrow_kernels-registry-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-core-arrow_kernels-registry-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-core-arrow_kernels-registry-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-core-arrow_kernels-registry-ut + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(ydb-library-yql-core-arrow_kernels-registry-ut) diff --git a/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.txt b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..55f8a282d87 --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/registry/ut/CMakeLists.windows-x86_64.txt @@ -0,0 +1,69 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-core-arrow_kernels-registry-ut) +target_compile_options(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry +) +target_link_libraries(ydb-library-yql-core-arrow_kernels-registry-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + core-arrow_kernels-registry + core-arrow_kernels-request + yql-public-udf + udf-service-exception_policy + yql-sql-pg_dummy + yql-minikql-invoke_builtins + yql-minikql-comp_nodes +) +target_sources(ydb-library-yql-core-arrow_kernels-registry-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp +) +set_property( + TARGET + ydb-library-yql-core-arrow_kernels-registry-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-core-arrow_kernels-registry-ut + TEST_TARGET + ydb-library-yql-core-arrow_kernels-registry-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-core-arrow_kernels-registry-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-core-arrow_kernels-registry-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-core-arrow_kernels-registry-ut + system_allocator +) +vcs_info(ydb-library-yql-core-arrow_kernels-registry-ut) diff --git a/ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp b/ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp new file mode 100644 index 00000000000..1acaa8637ba --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/registry/ut/registry_ut.cpp @@ -0,0 +1,84 @@ +#include <ydb/library/yql/core/arrow_kernels/request/request.h> +#include <ydb/library/yql/core/arrow_kernels/registry/registry.h> +#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> +#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h> + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NYql; +using namespace NKikimr::NMiniKQL; + +template <typename F> +void TestOne(F&& f) { + TExprContext ctx; + auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); + auto nodeFactory = GetBuiltinFactory(); + TKernelRequestBuilder b(*functionRegistry); + auto index = f(b, ctx); + UNIT_ASSERT_VALUES_EQUAL(index, 0); + auto s = b.Serialize(); + auto v = LoadKernels(s, *functionRegistry, nodeFactory); + UNIT_ASSERT_VALUES_EQUAL(v.size(), 1); +} + +Y_UNIT_TEST_SUITE(TKernelRegistryTest) { + Y_UNIT_TEST(TestZeroKernels) { + auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); + auto nodeFactory = GetBuiltinFactory(); + TKernelRequestBuilder b(*functionRegistry); + auto s = b.Serialize(); + auto v = LoadKernels(s, *functionRegistry, nodeFactory); + UNIT_ASSERT_VALUES_EQUAL(v.size(), 0); + } + + Y_UNIT_TEST(TestTwoKernels) { + TExprContext ctx; + auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); + auto nodeFactory = GetBuiltinFactory(); + TKernelRequestBuilder b(*functionRegistry); + auto blockInt32Type = ctx.template MakeType<TBlockExprType>(ctx.template MakeType<TDataExprType>(EDataSlot::Int32)); + auto index1 = b.AddBinaryOp(TKernelRequestBuilder::EBinaryOp::Add, blockInt32Type, blockInt32Type, blockInt32Type); + UNIT_ASSERT_VALUES_EQUAL(index1, 0); + auto index2 = b.AddBinaryOp(TKernelRequestBuilder::EBinaryOp::Sub, blockInt32Type, blockInt32Type, blockInt32Type); + UNIT_ASSERT_VALUES_EQUAL(index2, 1); + auto s = b.Serialize(); + auto v = LoadKernels(s, *functionRegistry, nodeFactory); + UNIT_ASSERT_VALUES_EQUAL(v.size(), 2); + } + + Y_UNIT_TEST(TestNot) { + TestOne([](auto& b,auto& ctx) { + auto blockBoolType = ctx.template MakeType<TBlockExprType>(ctx.template MakeType<TDataExprType>(EDataSlot::Bool)); + return b.AddUnaryOp(TKernelRequestBuilder::EUnaryOp::Not, blockBoolType, blockBoolType); + }); + } + + Y_UNIT_TEST(TestAdd) { + TestOne([](auto& b,auto& ctx) { + auto blockInt32Type = ctx.template MakeType<TBlockExprType>(ctx.template MakeType<TDataExprType>(EDataSlot::Int32)); + return b.AddBinaryOp(TKernelRequestBuilder::EBinaryOp::Add, blockInt32Type, blockInt32Type, blockInt32Type); + }); + } + + Y_UNIT_TEST(TestSub) { + TestOne([](auto& b,auto& ctx) { + auto blockInt32Type = ctx.template MakeType<TBlockExprType>(ctx.template MakeType<TDataExprType>(EDataSlot::Int32)); + return b.AddBinaryOp(TKernelRequestBuilder::EBinaryOp::Sub, blockInt32Type, blockInt32Type, blockInt32Type); + }); + } + + Y_UNIT_TEST(TestMul) { + TestOne([](auto& b,auto& ctx) { + auto blockInt32Type = ctx.template MakeType<TBlockExprType>(ctx.template MakeType<TDataExprType>(EDataSlot::Int32)); + return b.AddBinaryOp(TKernelRequestBuilder::EBinaryOp::Mul, blockInt32Type, blockInt32Type, blockInt32Type); + }); + } + + Y_UNIT_TEST(TestDiv) { + TestOne([](auto& b,auto& ctx) { + auto blockInt32Type = ctx.template MakeType<TBlockExprType>(ctx.template MakeType<TDataExprType>(EDataSlot::Int32)); + auto blockOptInt32Type = ctx.template MakeType<TBlockExprType>(ctx.template MakeType<TOptionalExprType>(ctx.template MakeType<TDataExprType>(EDataSlot::Int32))); + return b.AddBinaryOp(TKernelRequestBuilder::EBinaryOp::Div, blockInt32Type, blockInt32Type, blockOptInt32Type); + }); + } +} diff --git a/ydb/library/yql/core/arrow_kernels/request/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..5877718c2a4 --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,23 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(core-arrow_kernels-request) +target_compile_options(core-arrow_kernels-request PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(core-arrow_kernels-request PUBLIC + contrib-libs-cxxsupp + yutil + library-yql-ast + library-yql-minikql + providers-common-mkql +) +target_sources(core-arrow_kernels-request PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/request/request.cpp +) diff --git a/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..a38846dc53d --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-aarch64.txt @@ -0,0 +1,24 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(core-arrow_kernels-request) +target_compile_options(core-arrow_kernels-request PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(core-arrow_kernels-request PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-yql-ast + library-yql-minikql + providers-common-mkql +) +target_sources(core-arrow_kernels-request PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/request/request.cpp +) diff --git a/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..a38846dc53d --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.linux-x86_64.txt @@ -0,0 +1,24 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(core-arrow_kernels-request) +target_compile_options(core-arrow_kernels-request PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(core-arrow_kernels-request PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-yql-ast + library-yql-minikql + providers-common-mkql +) +target_sources(core-arrow_kernels-request PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/request/request.cpp +) diff --git a/ydb/library/yql/core/arrow_kernels/request/CMakeLists.txt b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/core/arrow_kernels/request/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..5877718c2a4 --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/request/CMakeLists.windows-x86_64.txt @@ -0,0 +1,23 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(core-arrow_kernels-request) +target_compile_options(core-arrow_kernels-request PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(core-arrow_kernels-request PUBLIC + contrib-libs-cxxsupp + yutil + library-yql-ast + library-yql-minikql + providers-common-mkql +) +target_sources(core-arrow_kernels-request PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/arrow_kernels/request/request.cpp +) diff --git a/ydb/library/yql/core/arrow_kernels/request/request.cpp b/ydb/library/yql/core/arrow_kernels/request/request.cpp new file mode 100644 index 00000000000..8450acbce97 --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/request/request.cpp @@ -0,0 +1,82 @@ +#include "request.h" +#include <ydb/library/yql/providers/common/mkql/yql_type_mkql.h> +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_node_serialization.h> + +namespace NYql { + +TKernelRequestBuilder::TKernelRequestBuilder(const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry) + : Alloc_(__LOCATION__) + , Env_(Alloc_) + , Pb_(Env_, functionRegistry) +{ + Alloc_.Release(); +} + +TKernelRequestBuilder::~TKernelRequestBuilder() { + Alloc_.Acquire(); +} + +ui32 TKernelRequestBuilder::AddUnaryOp(EUnaryOp op, const TTypeAnnotationNode* arg1Type, const TTypeAnnotationNode* retType) { + TGuard<NKikimr::NMiniKQL::TScopedAlloc> allocGuard(Alloc_); + auto returnType = MakeType(retType); + Y_UNUSED(returnType); + auto arg1 = MakeArg(arg1Type); + switch (op) { + case EUnaryOp::Not: + Items_.emplace_back(Pb_.BlockNot(arg1)); + break; + } + + return Items_.size() - 1; +} + +ui32 TKernelRequestBuilder::AddBinaryOp(EBinaryOp op, const TTypeAnnotationNode* arg1Type, const TTypeAnnotationNode* arg2Type, const TTypeAnnotationNode* retType) { + TGuard<NKikimr::NMiniKQL::TScopedAlloc> allocGuard(Alloc_); + auto returnType = MakeType(retType); + auto arg1 = MakeArg(arg1Type); + auto arg2 = MakeArg(arg2Type); + switch (op) { + case EBinaryOp::Add: + Items_.emplace_back(Pb_.BlockFunc("Add", returnType, { arg1, arg2 })); + break; + case EBinaryOp::Sub: + Items_.emplace_back(Pb_.BlockFunc("Sub", returnType, { arg1, arg2 })); + break; + case EBinaryOp::Mul: + Items_.emplace_back(Pb_.BlockFunc("Mul", returnType, { arg1, arg2 })); + break; + case EBinaryOp::Div: + Items_.emplace_back(Pb_.BlockFunc("Div", returnType, { arg1, arg2 })); + break; + } + + return Items_.size() - 1; +} + +TString TKernelRequestBuilder::Serialize() { + TGuard<NKikimr::NMiniKQL::TScopedAlloc> allocGuard(Alloc_); + auto tuple = Items_.empty() ? Pb_.AsScalar(Pb_.NewEmptyTuple()) : Pb_.BlockAsTuple(Items_); + return NKikimr::NMiniKQL::SerializeRuntimeNode(tuple, Env_); +} + +NKikimr::NMiniKQL::TRuntimeNode TKernelRequestBuilder::MakeArg(const TTypeAnnotationNode* type) { + auto [it, inserted] = CachedArgs_.emplace(type, NKikimr::NMiniKQL::TRuntimeNode()); + if (!inserted) { + return it->second; + } + + return it->second = Pb_.Arg(MakeType(type)); +} + +NKikimr::NMiniKQL::TBlockType* TKernelRequestBuilder::MakeType(const TTypeAnnotationNode* type) { + TStringStream err; + auto ret = NCommon::BuildType(*type, Pb_, err); + if (!ret) { + ythrow yexception() << err.Str(); + } + + return AS_TYPE(NKikimr::NMiniKQL::TBlockType, ret); +} + +} diff --git a/ydb/library/yql/core/arrow_kernels/request/request.h b/ydb/library/yql/core/arrow_kernels/request/request.h new file mode 100644 index 00000000000..f3eb860a0d4 --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/request/request.h @@ -0,0 +1,41 @@ +#pragma once +#include <ydb/library/yql/ast/yql_expr.h> +#include <ydb/library/yql/minikql/mkql_program_builder.h> + +#include <unordered_map> + +namespace NYql { + +class TKernelRequestBuilder { +public: + enum EUnaryOp { + Not + }; + + enum EBinaryOp { + Add, + Sub, + Mul, + Div + }; + + TKernelRequestBuilder(const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry); + ~TKernelRequestBuilder(); + + ui32 AddUnaryOp(EUnaryOp op, const TTypeAnnotationNode* arg1Type, const TTypeAnnotationNode* retType); + ui32 AddBinaryOp(EBinaryOp op, const TTypeAnnotationNode* arg1Type, const TTypeAnnotationNode* arg2Type, const TTypeAnnotationNode* retType); + TString Serialize(); + +private: + NKikimr::NMiniKQL::TRuntimeNode MakeArg(const TTypeAnnotationNode* type); + NKikimr::NMiniKQL::TBlockType* MakeType(const TTypeAnnotationNode* type); +private: + NKikimr::NMiniKQL::TScopedAlloc Alloc_; + NKikimr::NMiniKQL::TTypeEnvironment Env_; + NKikimr::NMiniKQL::TProgramBuilder Pb_; + TVector<NKikimr::NMiniKQL::TRuntimeNode> Items_; + std::unordered_map<const TTypeAnnotationNode*, NKikimr::NMiniKQL::TRuntimeNode> CachedArgs_; +}; + + +}
\ No newline at end of file diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_block_logical.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_block_logical.cpp index c33168c4108..e2cbcc9e845 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_block_logical.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_block_logical.cpp @@ -493,6 +493,49 @@ private: }; class TBlockNotWrapper : public TMutableComputationNode<TBlockNotWrapper> { +friend class TArrowNode; +public: + class TArrowNode : public IArrowKernelComputationNode { + public: + TArrowNode(const TBlockNotWrapper* parent) + : Parent_(parent) + , ArgsValuesDescr_({arrow::uint8()}) + , Kernel_({arrow::uint8()}, arrow::uint8(), [parent](arrow::compute::KernelContext* ctx, const arrow::compute::ExecBatch& batch, arrow::Datum* res) { + *res = parent->CalculateImpl(MakeDatumProvider(batch.values[0]), *ctx->memory_pool()); + return arrow::Status::OK(); + }) + { + Kernel_.null_handling = arrow::compute::NullHandling::COMPUTED_NO_PREALLOCATE; + Kernel_.mem_allocation = arrow::compute::MemAllocation::NO_PREALLOCATE; + } + + TStringBuf GetKernelName() const final { + return "Not"; + } + + const arrow::compute::ScalarKernel& GetArrowKernel() const { + return Kernel_; + } + + const std::vector<arrow::ValueDescr>& GetArgsDesc() const { + return ArgsValuesDescr_; + } + + const IComputationNode* GetArgument(ui32 index) const { + switch (index) { + case 0: + return Parent_->Value; + default: + throw yexception() << "Bad argument index"; + } + } + + private: + const TBlockNotWrapper* Parent_; + const std::vector<arrow::ValueDescr> ArgsValuesDescr_; + arrow::compute::ScalarKernel Kernel_; + }; + public: TBlockNotWrapper(TComputationMutables& mutables, IComputationNode* value) : TMutableComputationNode(mutables) @@ -500,25 +543,32 @@ public: { } - NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { - auto value = Value->GetValue(ctx); - const auto& datum = TArrowBlock::From(value).GetDatum(); + std::unique_ptr<IArrowKernelComputationNode> PrepareArrowKernelComputationNode(TComputationContext& ctx) const final { + Y_UNUSED(ctx); + return std::make_unique<TArrowNode>(this); + } + arrow::Datum CalculateImpl(const TDatumProvider& valueProv, arrow::MemoryPool& memoryPool) const { + auto datum = valueProv(); if (datum.null_count() == datum.length()) { - return value.Release(); + return datum; } if (datum.is_scalar()) { Y_VERIFY(datum.scalar()->is_valid); ui8 negated = (~datum.scalar_as<arrow::UInt8Scalar>().value) & 1u; - return ctx.HolderFactory.CreateArrowBlock(arrow::Datum(negated)); + return arrow::Datum(negated); } auto arr = datum.array(); - std::shared_ptr<arrow::Buffer> bitmap = CopyBitmap(&ctx.ArrowMemoryPool, arr->buffers[0], arr->offset, arr->length); - std::shared_ptr<arrow::Buffer> data = arrow::AllocateBuffer(arr->length, &ctx.ArrowMemoryPool).ValueOrDie(); + std::shared_ptr<arrow::Buffer> bitmap = CopyBitmap(&memoryPool, arr->buffers[0], arr->offset, arr->length); + std::shared_ptr<arrow::Buffer> data = arrow::AllocateBuffer(arr->length, &memoryPool).ValueOrDie(); NegateSparseBitmap(data->mutable_data(), arr->GetValues<ui8>(1), arr->length); - return ctx.HolderFactory.CreateArrowBlock(arrow::ArrayData::Make(arr->type, arr->length, { bitmap, data })); + return arrow::ArrayData::Make(arr->type, arr->length, { bitmap, data }); + } + + NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { + return ctx.HolderFactory.CreateArrowBlock(CalculateImpl(MakeDatumProvider(Value, ctx), ctx.ArrowMemoryPool)); } private: |