diff options
author | udovichenko-r <rvu@ydb.tech> | 2023-08-28 17:11:12 +0300 |
---|---|---|
committer | udovichenko-r <rvu@ydb.tech> | 2023-08-28 17:49:09 +0300 |
commit | 4e949f902f6ad6f88dbd27508547fd04ccfc413a (patch) | |
tree | 20fb5159bf96e16d166d7366123752a558b88532 | |
parent | 1e82f33abca18eecf6a334a3a20c81398765b1a2 (diff) | |
download | ydb-4e949f902f6ad6f88dbd27508547fd04ccfc413a.tar.gz |
[yql] cleanup ya.makes (part 1)
YQL-16404
283 files changed, 44288 insertions, 54 deletions
diff --git a/contrib/libs/cxxsupp/libcxx/include/semaphore b/contrib/libs/cxxsupp/libcxx/include/semaphore new file mode 100644 index 00000000000..753d50f5126 --- /dev/null +++ b/contrib/libs/cxxsupp/libcxx/include/semaphore @@ -0,0 +1,188 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP_SEMAPHORE +#define _LIBCPP_SEMAPHORE + +/* + semaphore synopsis + +namespace std { + +template<ptrdiff_t least_max_value = implementation-defined> +class counting_semaphore +{ +public: +static constexpr ptrdiff_t max() noexcept; + +constexpr explicit counting_semaphore(ptrdiff_t desired); +~counting_semaphore(); + +counting_semaphore(const counting_semaphore&) = delete; +counting_semaphore& operator=(const counting_semaphore&) = delete; + +void release(ptrdiff_t update = 1); +void acquire(); +bool try_acquire() noexcept; +template<class Rep, class Period> + bool try_acquire_for(const chrono::duration<Rep, Period>& rel_time); +template<class Clock, class Duration> + bool try_acquire_until(const chrono::time_point<Clock, Duration>& abs_time); + +private: +ptrdiff_t counter; // exposition only +}; + +using binary_semaphore = counting_semaphore<1>; + +} + +*/ + +#include <__availability> +#include <__config> +#include <__thread/timed_backoff_policy.h> +#include <__threading_support> +#include <atomic> +#include <version> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#ifdef _LIBCPP_HAS_NO_THREADS +# error <semaphore> is not supported on this single threaded system +#endif + +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + +#if _LIBCPP_STD_VER >= 14 + +_LIBCPP_BEGIN_NAMESPACE_STD + +/* + +__atomic_semaphore_base is the general-case implementation. +It is a typical Dijkstra semaphore algorithm over atomics, wait and notify +functions. It avoids contention against users' own use of those facilities. + +*/ + +class __atomic_semaphore_base +{ + __atomic_base<ptrdiff_t> __a; + +public: + _LIBCPP_INLINE_VISIBILITY + constexpr explicit __atomic_semaphore_base(ptrdiff_t __count) : __a(__count) + { + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY + void release(ptrdiff_t __update = 1) + { + if(0 < __a.fetch_add(__update, memory_order_release)) + ; + else if(__update > 1) + __a.notify_all(); + else + __a.notify_one(); + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY + void acquire() + { + auto const __test_fn = [this]() -> bool { + auto __old = __a.load(memory_order_relaxed); + return (__old != 0) && __a.compare_exchange_strong(__old, __old - 1, memory_order_acquire, memory_order_relaxed); + }; + __cxx_atomic_wait(&__a.__a_, __test_fn); + } + template <class Rep, class Period> + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY + bool try_acquire_for(chrono::duration<Rep, Period> const& __rel_time) + { + if (__rel_time == chrono::duration<Rep, Period>::zero()) + return try_acquire(); + auto const __test_fn = [this]() { return try_acquire(); }; + return __libcpp_thread_poll_with_backoff(__test_fn, __libcpp_timed_backoff_policy(), __rel_time); + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY + bool try_acquire() + { + auto __old = __a.load(memory_order_acquire); + while (true) { + if (__old == 0) + return false; + if (__a.compare_exchange_strong(__old, __old - 1, memory_order_acquire, memory_order_relaxed)) + return true; + } + } +}; + +#define _LIBCPP_SEMAPHORE_MAX (numeric_limits<ptrdiff_t>::max()) + +template<ptrdiff_t __least_max_value = _LIBCPP_SEMAPHORE_MAX> +class counting_semaphore +{ + __atomic_semaphore_base __semaphore; + +public: + static constexpr ptrdiff_t max() noexcept { + return __least_max_value; + } + + _LIBCPP_INLINE_VISIBILITY + constexpr explicit counting_semaphore(ptrdiff_t __count) : __semaphore(__count) { } + ~counting_semaphore() = default; + + counting_semaphore(const counting_semaphore&) = delete; + counting_semaphore& operator=(const counting_semaphore&) = delete; + + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY + void release(ptrdiff_t __update = 1) + { + __semaphore.release(__update); + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY + void acquire() + { + __semaphore.acquire(); + } + template<class Rep, class Period> + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY + bool try_acquire_for(chrono::duration<Rep, Period> const& __rel_time) + { + return __semaphore.try_acquire_for(chrono::duration_cast<chrono::nanoseconds>(__rel_time)); + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY + bool try_acquire() + { + return __semaphore.try_acquire(); + } + template <class Clock, class Duration> + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY + bool try_acquire_until(chrono::time_point<Clock, Duration> const& __abs_time) + { + auto const current = Clock::now(); + if (current >= __abs_time) + return try_acquire(); + else + return try_acquire_for(__abs_time - current); + } +}; + +using binary_semaphore = counting_semaphore<1>; + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP_STD_VER >= 14 + +_LIBCPP_POP_MACROS + +#endif //_LIBCPP_SEMAPHORE diff --git a/contrib/libs/protobuf/python/ya.make b/contrib/libs/protobuf/python/ya.make new file mode 100644 index 00000000000..9031af5cbba --- /dev/null +++ b/contrib/libs/protobuf/python/ya.make @@ -0,0 +1,9 @@ +PACKAGE() + +WITHOUT_LICENSE_TEXTS() + +LICENSE(BSD-3-Clause) + +GENERATE_PY_PROTOS(contrib/libs/protobuf/src/google/protobuf/descriptor.proto) + +END() diff --git a/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt index 1101ee7d42f..a29f25c4c65 100644 --- a/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt @@ -18,6 +18,7 @@ add_subdirectory(file_storage) add_subdirectory(issue) add_subdirectory(peephole_opt) add_subdirectory(services) +add_subdirectory(spilling) add_subdirectory(sql_types) add_subdirectory(type_ann) add_subdirectory(url_lister) diff --git a/ydb/library/yql/core/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/CMakeLists.linux-aarch64.txt index e70610f312a..ad218bf3183 100644 --- a/ydb/library/yql/core/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/core/CMakeLists.linux-aarch64.txt @@ -18,6 +18,7 @@ add_subdirectory(file_storage) add_subdirectory(issue) add_subdirectory(peephole_opt) add_subdirectory(services) +add_subdirectory(spilling) add_subdirectory(sql_types) add_subdirectory(type_ann) add_subdirectory(url_lister) diff --git a/ydb/library/yql/core/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/CMakeLists.linux-x86_64.txt index e70610f312a..ad218bf3183 100644 --- a/ydb/library/yql/core/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/core/CMakeLists.linux-x86_64.txt @@ -18,6 +18,7 @@ add_subdirectory(file_storage) add_subdirectory(issue) add_subdirectory(peephole_opt) add_subdirectory(services) +add_subdirectory(spilling) add_subdirectory(sql_types) add_subdirectory(type_ann) add_subdirectory(url_lister) diff --git a/ydb/library/yql/core/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/CMakeLists.windows-x86_64.txt index 1101ee7d42f..a29f25c4c65 100644 --- a/ydb/library/yql/core/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/core/CMakeLists.windows-x86_64.txt @@ -18,6 +18,7 @@ add_subdirectory(file_storage) add_subdirectory(issue) add_subdirectory(peephole_opt) add_subdirectory(services) +add_subdirectory(spilling) add_subdirectory(sql_types) add_subdirectory(type_ann) add_subdirectory(url_lister) diff --git a/ydb/library/yql/core/arrow_kernels/ya.make b/ydb/library/yql/core/arrow_kernels/ya.make new file mode 100644 index 00000000000..9e858039fc6 --- /dev/null +++ b/ydb/library/yql/core/arrow_kernels/ya.make @@ -0,0 +1,4 @@ +RECURSE( + registry + request +) diff --git a/ydb/library/yql/core/services/ya.make b/ydb/library/yql/core/services/ya.make index 45af1e24ff7..9a877d25514 100644 --- a/ydb/library/yql/core/services/ya.make +++ b/ydb/library/yql/core/services/ya.make @@ -36,3 +36,7 @@ PEERDIR( YQL_LAST_ABI_VERSION() END() + +RECURSE( + mounts +) diff --git a/ydb/library/yql/core/spilling/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/spilling/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..e222b22bb9b --- /dev/null +++ b/ydb/library/yql/core/spilling/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,39 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(storage) +add_subdirectory(ut) + +add_library(yql-core-spilling) +target_compile_options(yql-core-spilling PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION + $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything> +) +target_link_libraries(yql-core-spilling PUBLIC + contrib-libs-cxxsupp + yutil + libs-apache-arrow + ydb-library-binary_json + library-yql-utils + yql-utils-log + core-spilling-storage + yql-minikql-codegen + llvm12-lib-IR + lib-ExecutionEngine-MCJIT + llvm12-lib-Linker + lib-Target-X86 + Target-X86-AsmParser + lib-Transforms-IPO +) +target_sources(yql-core-spilling PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/spilling_imp.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/namespaces_list.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/namespace_cache.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/storage.cpp +) diff --git a/ydb/library/yql/core/spilling/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/spilling/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..de3eadabe42 --- /dev/null +++ b/ydb/library/yql/core/spilling/CMakeLists.linux-aarch64.txt @@ -0,0 +1,40 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(storage) +add_subdirectory(ut) + +add_library(yql-core-spilling) +target_compile_options(yql-core-spilling PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION + $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything> +) +target_link_libraries(yql-core-spilling PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + libs-apache-arrow + ydb-library-binary_json + library-yql-utils + yql-utils-log + core-spilling-storage + yql-minikql-codegen + llvm12-lib-IR + lib-ExecutionEngine-MCJIT + llvm12-lib-Linker + lib-Target-X86 + Target-X86-AsmParser + lib-Transforms-IPO +) +target_sources(yql-core-spilling PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/spilling_imp.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/namespaces_list.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/namespace_cache.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/storage.cpp +) diff --git a/ydb/library/yql/core/spilling/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/spilling/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..de3eadabe42 --- /dev/null +++ b/ydb/library/yql/core/spilling/CMakeLists.linux-x86_64.txt @@ -0,0 +1,40 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(storage) +add_subdirectory(ut) + +add_library(yql-core-spilling) +target_compile_options(yql-core-spilling PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION + $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything> +) +target_link_libraries(yql-core-spilling PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + libs-apache-arrow + ydb-library-binary_json + library-yql-utils + yql-utils-log + core-spilling-storage + yql-minikql-codegen + llvm12-lib-IR + lib-ExecutionEngine-MCJIT + llvm12-lib-Linker + lib-Target-X86 + Target-X86-AsmParser + lib-Transforms-IPO +) +target_sources(yql-core-spilling PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/spilling_imp.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/namespaces_list.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/namespace_cache.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/storage.cpp +) diff --git a/ydb/library/yql/core/spilling/CMakeLists.txt b/ydb/library/yql/core/spilling/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/core/spilling/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/core/spilling/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/spilling/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..e222b22bb9b --- /dev/null +++ b/ydb/library/yql/core/spilling/CMakeLists.windows-x86_64.txt @@ -0,0 +1,39 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(storage) +add_subdirectory(ut) + +add_library(yql-core-spilling) +target_compile_options(yql-core-spilling PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION + $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything> +) +target_link_libraries(yql-core-spilling PUBLIC + contrib-libs-cxxsupp + yutil + libs-apache-arrow + ydb-library-binary_json + library-yql-utils + yql-utils-log + core-spilling-storage + yql-minikql-codegen + llvm12-lib-IR + lib-ExecutionEngine-MCJIT + llvm12-lib-Linker + lib-Target-X86 + Target-X86-AsmParser + lib-Transforms-IPO +) +target_sources(yql-core-spilling PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/spilling_imp.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/namespaces_list.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/namespace_cache.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/storage.cpp +) diff --git a/ydb/library/yql/core/spilling/interface/spilling.h b/ydb/library/yql/core/spilling/interface/spilling.h new file mode 100644 index 00000000000..1d32d3b110e --- /dev/null +++ b/ydb/library/yql/core/spilling/interface/spilling.h @@ -0,0 +1,195 @@ +#pragma once + +#include <library/cpp/threading/future/async.h> +#include <util/generic/buffer.h> + +namespace NYql { +namespace NSpilling { + +struct TFileStorageConfig; // Configuration attributes to instantiate TempStorageProxy stored in filesystem folder +struct TTempStorageExecutionPolicy; // Execution policy to work with temporary storage +struct TTempObjectDesc; // Represents description of particular objects in temporary storage +class ITempStorageProxy; // Proxy class to provide unified interface for all types of temporary storage +struct TOperationResults; // Contains information about particular operation results + +// Factory method to create TempStorageProxy for usage. Error reasons are returned in TOperatonResults +std::pair<THolder<ITempStorageProxy>, TOperationResults> CreateFileStorageProxy(const TFileStorageConfig& config, const TTempStorageExecutionPolicy& policy ); + +struct TLoadOperationResults; // Contains information about load operation together with loaded data + +class ISession; // Session to manage temp objects lifecycle for particular process +struct TSessionExecutionPolicy; // Execution policy for session to redefine default TTempStorageExecutionPolicy settings +class IStream; // Class to save and load temp objects to storage in stream mode. Stream serializes objects order as it was saved, allows random + // access to stream objects by object index and provides consistency guarantee for all objects in stream when Close call succeed. + + +// Iterator over temp storage objects +class IObjectsIterator { +public: + + virtual bool Next(TTempObjectDesc& object) = 0; // Fills next temporary object description. Returns true if there are more objects + virtual ~IObjectsIterator() = default; +}; + + +// Amount of data processed per session +struct TSessionDataStat { + ui64 Provided = 0; // Total data provided by client to save in session (MB) + ui64 InMemory = 0; // Amount on data for session in memory (MB) + ui64 Spilled = 0; // Session data spilled to storage or deleted in memory (MB) + ui64 LoadedFromStorage = 0; // Amount of data loaded from storage (MB) + ui64 LoadedFromMemory = 0; // Session data processed in-memory without spilling (MB) +}; + + +// Possible lifetime management options for objects stored in particular session +enum class EObjectsLifetime { + DeleteAfterLoad = 0, // Object is deleted automatically from spilling storage after load operation. Object ownership is on the caller side after load call. + SharedForSession = 1, // Object can be loaded many times during session and object memory is managed by shared pointer. Pointer is shared between spilling Session and all callers for the object. + Persistent = 2, // Object is persistent and stored in spilling storage until deleted + }; + + +class ISession { +public: + // Saves buf with size bytes to temporary storage. buf content should not be changed until operation completes. + // TOperationResults contains final results of buffer saving. buf is deleted after saving. + virtual NThreading::TFuture<TOperationResults> Save(const TString& objNamespace, const TString& name, TBuffer&& buf) = 0; + + // Loads data from temporary storage to Buf in TLoadOperationResults. + virtual NThreading::TFuture<TLoadOperationResults> Load(const TString& objNamespace, const TString& name, EObjectsLifetime objLifetime = EObjectsLifetime::DeleteAfterLoad) = 0; + + // Return current size of all temporary buffers to save. It should be used to control speed of Save operations from the client side + virtual TSessionDataStat GetSessionDataStat() = 0; + + virtual TSessionExecutionPolicy ExecutionPolicy() = 0; // Returns current execution policy of ISession + virtual TOperationResults SetExecutionPolicy(const TSessionExecutionPolicy& policy) = 0; // Changes execution policy of ISession + + // Opens stream to save or load data. Possible errors are returned in TOperationResults + virtual std::pair<THolder<IStream>, TOperationResults> OpenStream(const TString& objNamespace, const TString& streamName ) = 0; + + virtual ~ISession() = default; +}; + +class IStream { +public: + // Saves buf to stream. Buffer id in stream is returned in TOperationResults. Buffer id is a 0,1,2,3..(Size of stream - 1) sequence. + // Stream guarantees buffer id order the same as it was Save calls order. + virtual NThreading::TFuture<TOperationResults> Save(TBuffer&& buf) = 0; + + // Return current number of objects stored in stream. + virtual ui64 Size() = 0; + + // Loads data from stream with provided buffer id + virtual NThreading::TFuture<TLoadOperationResults> Load(ui64 bufferId = 0, EObjectsLifetime objLifetime = EObjectsLifetime::DeleteAfterLoad) = 0; + + // Closes stream to ensure consistensy of all stream data buffers during subsequent load operations. + virtual NThreading::TFuture<TOperationResults> Close() = 0; + + virtual ~IStream() = default; +}; + + + + + +class ITempStorageProxy { +public: + + // Creates new session to store and load temporary objects; Session manages lifecycle of all resources associated with the session. + // When session is deleted, all pending load operations are canceled, all resources associated with the session are freed + virtual THolder<ISession> CreateSession() = 0; + + // Creates iterator to enumerate stored objects of interest. It works both for namespaces and objects enumeration. + // Objects are identified by namespace and name. If onlyValid = true, only valid objects are returned + virtual THolder<IObjectsIterator> CreateIterator( const TMaybe<TString>& objNamespace = TMaybe<TString>(), + const TMaybe<TString>& objName = TMaybe<TString>(), + bool onlyValid = true) = 0; + + + virtual TTempStorageExecutionPolicy ExecutionPolicy() = 0; // Returns current execution policy of ITempStorageProxy + virtual TOperationResults SetExecutionPolicy(const TTempStorageExecutionPolicy& policy) = 0; // Changes execution policy of ITempStorageProxy + + // Deletes object with particular namespace and name. If name is empty, all objects from particular namespace are deleted. + virtual NThreading::TFuture<TOperationResults> Delete(const TString& objNamespace, const TMaybe<TString>& name) = 0; + + virtual ~ITempStorageProxy() = default; + +}; + + +struct TTempStorageExecutionPolicy { + ui64 MaxStorageSize = 100; // Maximim size of temporary storage in GB. We are trying to keep total size of temporary objects withing this limit + ui64 RetentionPeriod = 24; // Default retention period in hours for temporary objects + ui64 DeleteOnClose = true; // When true, all temporary objects are deleted automatically when Session object is closed + bool CanClearNow = true; // If true, background deletion of obsolete objects is performed + ui64 MaxBandwidth = 10 * 1000; // ( Maximim bandwidth limitation for particular TempStorageProxy in MB/sec. Default is 10 GB/sec ) + ui64 MaxCallsPerSecond = 100; // Maximim calls per second for particular TempStorageProxy + ui64 OneCallTimeout = 10; // Default timeout for one call to storage in seconds. + bool RetryCall = true; // If true, operation we are trying to complete operation using retry policy settings + ui64 RetryPeriod = 2; // Default period for retry calls in seconds + bool DoubleRetryPeriod = true; // Doubles retry period util it reaches MaxRetryPeriod; + ui64 MaxRetryPeriod = 60; // Maximim retry period in seconds + ui64 MaxNumberOfRetries = 10; // Maximim number of retries to complete operation. + ui64 MaxBuffersSize = 2000; // Total size of all process internals buffers in MB waiting either to load or store. + // When this limit exceeded, Save and Load operations starts returning BuffersAreFull error +}; + +struct TSessionExecutionPolicy { + ui64 DeleteOnClose = true; // When true, all temporary objects are deleted automatically when Session object is closed + ui64 MaxBuffersSize = 100; // Total size of session internals buffers in MB waiting either to load or store. + // When this limit exceeded, Save and Load operations starts returning BuffersAreFull error +}; + +// Possible reasons why object is considered invalid +enum class EBadObjectReason { + NoBadReason = 0, + ChecksumInvalid = 1, + SizeInvalid = 2, + NoNamespace = 3, + NoObjectName = 4, + ReadError = 5 + }; + +struct TTempObjectDesc { + TString Namespace; // Namespace name. Empty for global namespace + TString Name; // Object name. Empty for namespaces. For stream objects it is a name of the stream + ui64 Size; // Size of the object in bytes. Total size of the objects in namespace in case of namespaces + ui64 Index; // Index of the object in the stream, starting from 0 + bool IsValid; // True if object seems to be correct. Final consistency check is performed during full object load + bool IsFinal; // True if object is final in the stream + bool StreamValid; // True if stream is closed correctly + EBadObjectReason BadObjectReason; // Contains reason why object is considered invalid +}; + +// Possible statuses of operation +enum class EOperationStatus { + Success = 0, + Failure = 1, + CannotOpenStorageProxy = 2, + WrongBufferId = 3, + ProxyTerminated = 4, + BuffersAreFull = 5, + NoObjectName = 6, + NewVersionDefined = 7, + ChecksumInvalid = 8 +}; + +// Results of operation +struct TOperationResults { + EOperationStatus Status; // Final status of operation + ui64 BufferId; // Buffer id for stream buffers sequence + TString ErrorString; // Contains error description for logs and following diagnostics +}; + +// Results of load operation +struct TLoadOperationResults: public TOperationResults { + TAtomicSharedPtr<TBuffer> Buf; // Buffer with data after load operation. +}; + +struct TFileStorageConfig { + TString Path; // Storage path location +}; + +} +}
\ No newline at end of file diff --git a/ydb/library/yql/core/spilling/namespace_cache.cpp b/ydb/library/yql/core/spilling/namespace_cache.cpp new file mode 100644 index 00000000000..9faa9fbe9bb --- /dev/null +++ b/ydb/library/yql/core/spilling/namespace_cache.cpp @@ -0,0 +1,824 @@ +#include "namespaces_list.h" +#include "interface/spilling.h" +#include "storage/storage.h" + +#include <ydb/library/yql/utils/log/log.h> + +#include <map> + +#include <util/string/split.h> +#include <contrib/libs/xxhash/xxhash.h> + + +namespace NYql { +namespace NSpilling { + +// Finds approximate position of val in array of N increasing values. Values cycle after max(ui32) +// Returns true if val can be between start and end taking into account cycling. +inline bool FindPos(ui32 start, ui32 end, ui32 val, ui32 nElements, ui32 & pos) { + if (!nElements) + return false; + + if (start == end) { + if ( val == start ) { + pos = 0; + return true; + } else { + return false; + } + } + + if (start > end) { + ui32 len = (std::numeric_limits<ui32>::max() - start) + end; + if ((val >= start) || (val <= end)) { + pos = len / nElements; + return true; + } else { + return false; + } + } + + if ((val >= start) && (val <= end)) { + pos = ( (val - start) * (nElements - 1 ) ) / (end - start); + return true; + } else { + return false; + } + + return false; + +}; + +// Returns file index for particular object id +inline ui32 FindFileInd(ui32 objId) { + + ui32 fileId = (objId) / (1<<16); + return fileId; + +} + +inline void TNamespaceCache::UpdateMinSessionId(ui32 sessionId, ui32 objId) { + auto sessionIt = Sessions_.find(sessionId); + if ( sessionIt == Sessions_.end() ) { + Sessions_[sessionId].MinObjId = objId; + } else { + if (sessionIt->second.MinObjId > objId) { + sessionIt->second.MinObjId = objId; + } + } +} + + +NThreading::TFuture<TOperationResults> TNamespaceCache::Save(const TString & objName, TBuffer && buf, ui32 sessionId, bool isStream ) { + NThreading::TFuture<TOperationResults> res; + TSaveTask bt; + bt.Name = objName; + bt.SessionId = sessionId; + bt.Buf = MakeAtomicShared<TBuffer>(std::move(buf)); + + bt.Promise = NThreading::NewPromise<TOperationResults>(); + + if ( !CurrSpillMetaFile_ ) { + NextNamespaceFile(true); + } + + + with_lock(ToSaveLock_) { + AdvanceObjId(bt.ObjId); + if ( !isStream ) { + ui32 lastObjId = 0; + auto search = LastObjs_.find(objName); + if (search != LastObjs_.end() ) { + lastObjId = search->second; + } + if ( lastObjId ) { + ui32 bytesDeleted = DeleteTaskFromSaveQueue(lastObjId); + SessionDataSpilled_[sessionId] += bytesDeleted; + } + LastObjs_[objName] = bt.ObjId; + } else { + auto search = LastStreamIds_.find(objName); + ui32 streamInd = 0; + if (search != LastStreamIds_.end() ) { + streamInd = search->second.size(); + search->second.push_back(bt.ObjId); + } else { + streamInd = 0; + LastStreamIds_[objName].push_back(bt.ObjId); + } + bt.StreamBufId = streamInd; + bt.OpType = EOperationType::StreamBufAdd; + } + UpdateMinSessionId(sessionId, bt.ObjId); + SessionDataProvided_[sessionId] += bt.Buf->Size(); + res = bt.Promise.GetFuture(); + ToSave_.emplace_back(std::move(bt)); + } + return res; +} + + +NThreading::TFuture<TLoadOperationResults> TNamespaceCache::Load(const TString& name, ui32 sessionId, EObjectsLifetime objLifetime, bool isStream, ui32 streamId ) { + NThreading::TFuture<TLoadOperationResults> res; + TLoadTask lt; + TLoadOperationResults lr; + lt.Name = name; + lt.Promise = NThreading::NewPromise<TLoadOperationResults>(); + lt.Name = name; + lt.SessionId = sessionId; + lt.StreamBufId = streamId; + bool foundInLastObjs = false; + bool foundInSaveQueue = false; + with_lock(ToSaveLock_) { + if (!isStream) { + auto search = LastObjs_.find(name); + foundInLastObjs = (search != LastObjs_.end()); + lt.ObjId = search->second; + } else { + auto search = LastStreamIds_.find(name); + if (search != LastStreamIds_.end()) { + if ( search->second.size() >= streamId ) { + foundInLastObjs = true; + lt.ObjId = search->second[streamId]; + } + } + } + if ( foundInLastObjs ) { + UpdateMinSessionId(sessionId, lt.ObjId); + ui32 objPos = 0; + foundInSaveQueue = FindObjInSaveQueue(lt.ObjId, objPos); + if (foundInSaveQueue) { + TSaveTask & st = ToSave_[objPos]; + if ( st.ProcessingStatus != EProcessingStatus::Deleted ) { + lr.Status = EOperationStatus::Success; + if ( st.ProcessingStatus == EProcessingStatus::Processing ) { + lr.Buf = st.Buf; + } else { + if (objLifetime == EObjectsLifetime::DeleteAfterLoad ) { + lr.Buf = st.Buf; + st.Buf = nullptr; + SessionDataSpilled_[st.SessionId] += lr.Buf->Size(); + st.ProcessingStatus = EProcessingStatus::Deleted; + } else { + lr.Buf = st.Buf; + } + + } + SessionDataLoadedFromMemory_[st.SessionId] += lr.Buf->Size(); + lt.Promise.SetValue(std::move(lr)); + } else { + lr.Status = EOperationStatus::NoObjectName; + lt.Promise.SetValue(std::move(lr)); + } + } + } else { + lr.Status = EOperationStatus::NoObjectName; + lt.Promise.SetValue(std::move(lr)); + } + } + + res = lt.Promise.GetFuture(); + if (!foundInSaveQueue && foundInLastObjs) { + with_lock(ToLoadLock_) { + ToLoad_.emplace_back(std::move(lt)); + } + } + + return res; + +} + + +void TNamespaceCache::Dispatch() { + + ProcessSaveQueue(); + ProcessLoadQueue(); + ProcessDeleteQueue(); + +} + + +TSessionDataStat TNamespaceCache::GetSessionStat(ui32 sessionId) { + TSessionDataStat res; + ui64 provided = 0; + ui64 spilled = 0; + ui64 loadedFromMemory = 0; + ui64 loadedFromStorage = 0; + + with_lock(ToSaveLock_) { + auto it = SessionDataProvided_.find(sessionId); + if ( it != SessionDataProvided_.end()) { + provided = it->second; + res.Provided = provided / (1024 * 1024); + }; + it = SessionDataSpilled_.find(sessionId); + if (it != SessionDataProvided_.end()) { + spilled = it->second; + res.Spilled = spilled / (1024 * 1024); + }; + } + + with_lock(ToLoadLock_) { + auto it = SessionDataLoadedFromMemory_.find(sessionId); + if (it != SessionDataLoadedFromMemory_.end()) { + loadedFromMemory = it->second; + res.LoadedFromMemory = loadedFromMemory / (1024 * 1024); + }; + + it = SessionDataLoadedFromStorage_.find(sessionId); + if (it != SessionDataLoadedFromStorage_.end()) { + loadedFromStorage = it->second; + res.LoadedFromStorage = loadedFromStorage / (1024 * 1024); + }; + + + } + + ui64 totalProcessed = spilled; + if ( totalProcessed > provided ) { + res.InMemory = 0; + } else { + res.InMemory = (provided - totalProcessed) / (1024 * 1024); + } + + return res; +} + + +void TNamespaceCache::CloseSession(ui32 sessionId){ + TBaseTask dt; + dt.OpType = EOperationType::SessionDelete; + dt.SessionId = sessionId; + + with_lock(ToDeleteLock_) { + ToDelete_.emplace_back(dt); + } + +} + +void TNamespaceCache::GarbageCollection(){ + std::set<ui32> filesIdToDelete; + + ui32 minObjIdForSession = 0; + with_lock(ToSaveLock_) { + if (Sessions_.size() > 0) { + minObjIdForSession = Sessions_.begin()->second.MinObjId; + } + for (auto & s: Sessions_ ) { + if (s.second.MinObjId < minObjIdForSession ) { + minObjIdForSession = s.second.MinObjId; + } + } + + for ( auto it = LastObjs_.begin(); it != LastObjs_.end(); ) { + if (it->second < minObjIdForSession ) { + it = LastObjs_.erase(it); + } else { + it++; + } + } + + for ( auto it = LastStreamIds_.begin(); it != LastStreamIds_.end(); ) { + if ( it->second.size() > 0) { + if (it->second.front() < minObjIdForSession && it->second.back() < minObjIdForSession) { + it = LastStreamIds_.erase(it); + continue; + } + } + it++; + } + + + } + + + ui32 minFileId = FindFileInd(minObjIdForSession); + with_lock(FilesLock_) { + for (auto it = SpillMetaFiles_.begin(); it != SpillMetaFiles_.lower_bound(minFileId); it++ ) { + SpillFilesIdToDelete_.insert(it->first); + } + } + + + + with_lock(FilesLock_) { + filesIdToDelete = SpillFilesIdToDelete_; + + } + + for (auto id: filesIdToDelete) { + TAtomicSharedPtr<TMetaFileAttributes> met; + with_lock(FilesLock_) { + InitMetaFile(id); + met = SpillMetaFiles_[id]; + } + if ( met->DataFile->IsLocked()) { + met->DataFile->Delete(); + } + + if ( met->MetaFile->IsLocked()) { + met->MetaFile->Delete(); + } + } + + with_lock(FilesLock_) { + for (auto id: filesIdToDelete) { + SpillFilesIdToDelete_.erase(id); + SpillMetaFiles_.erase(id); + + } + + } + +} + + + +ui64 TNamespaceCache::StreamSize(const TString& name){ + with_lock(ToSaveLock_) { + auto search = LastStreamIds_.find(name); + if (search != LastStreamIds_.end()) { + return search->second.size(); + } else { + return 0; + } + } + +} + + +void TNamespaceCache::NextNamespaceFile(bool openExisting) { + with_lock(FilesLock_) { + TVector<TString> files = StorageI_->GetNamespaceFiles(Name_); + ui32 maxInd = 0; + for (const auto & p: files) { + TVector<TString> splitted; + size_t tokenCount = Split(p, ".", splitted); + if (tokenCount == 4 && splitted[0] == "ydbspl" ) { + ui32 ind = 0; + if (TryFromString(splitted[1], ind)) { + if (SpillMetaFiles_.find(ind) == SpillMetaFiles_.end() ) { + SpillFilesIdToDelete_.insert(ind); + } + if (ind > maxInd) { + maxInd = ind; + } + } + } + } + + + if (maxInd > std::numeric_limits<ui32>::max() / 2 ) { + maxInd = 1; + } + + SpillFilesIdToDelete_.erase(maxInd); + + TTempStorageExecutionPolicy pol; + ui32 retryCount = 0; + for (; retryCount < pol.MaxNumberOfRetries; retryCount++ ) { + THolder<ISpillFile> metFile = StorageI_->CreateSpillFile(Name_, TString("ydbspl.") + std::to_string(maxInd) + TString(".0.met"), MetaFileIncreaseStep); + if (metFile->IsLocked()) { + CurrSpillFileId_ = maxInd; + CurrSpillMetaFile_ = std::move(metFile); + THolder<ISpillFile> datFile = StorageI_->CreateSpillFile(Name_, TString("ydbspl.") + std::to_string(maxInd) + TString(".0.dat"), DataFileIncreaseStep); + CurrSpillDataFile_ = std::move(datFile); + NextObjId_ = CurrSpillFileId_ * (1<<16) + 1; + TAtomicSharedPtr<TMetaFileAttributes> res = MakeAtomicShared<TMetaFileAttributes>(); + res->MetaFile = CurrSpillMetaFile_; + res->DataFile = CurrSpillDataFile_; + res->Id = CurrSpillFileId_; + res->FirstObjId = NextObjId_; + res->MetaRecords = MakeAtomicShared<std::map<ui32, TSpillMetaRecord>>(); + SpillMetaFiles_[CurrSpillFileId_] = res; + break; + } else { + maxInd++; + } + } + + } + + +} + +inline void TNamespaceCache::AdvanceObjId(ui32 & objId) { + if (NextObjId_ == std::numeric_limits<ui32>::max() ) { + NextObjId_ = 1; + } + objId = ++NextObjId_; +} + + +inline bool TNamespaceCache::FindObjInSaveQueue(ui32 objId, ui32& pos){ + ui32 size = ToSave_.size(); + if ( size == 0) + return false; + ui32 firstId = ToSave_.front().ObjId; + ui32 lastId = ToSave_.back().ObjId; + ui32 approximatePos = 0; + bool found = FindPos(firstId, lastId, objId, size, approximatePos ); + if (!found) + return false; + + pos = approximatePos; + auto it = ToSave_.begin() + pos; + ui32 toSaveId = it->ObjId; + + if (toSaveId == objId) { + return true; + } + + if (toSaveId < objId) { + for( ; it != ToSave_.end(); it++) { + toSaveId = it->ObjId; + if (toSaveId == objId) { + pos = (it - ToSave_.begin()); + return true; + } + } + return false; + } + + if (toSaveId > objId) { + while(pos > 0) { + --pos; + if (ToSave_[pos].ObjId == objId) { + return true; + } + } + return false; + } + + return false; +} + + +ui32 TNamespaceCache::DeleteTaskFromSaveQueue(ui32 objId, EProcessingStatus reason) { + ui32 pos; + ui32 bytesDeleted = 0; + bool found = FindObjInSaveQueue(objId, pos); + if ( found && ToSave_[pos].ProcessingStatus != EProcessingStatus::Processing) { + ToSave_[pos].ProcessingStatus = reason; + bytesDeleted = ToSave_[pos].Buf->Size(); + ToSave_[pos].Buf = nullptr; + } + return bytesDeleted; + +} + +bool TNamespaceCache::FindNextTaskInSaveQueue(ui32& taskPos) { + + bool res = false; + for ( taskPos = 0; taskPos < ToSave_.size(); taskPos++) { + TSaveTask& saveTask = ToSave_[taskPos]; + if ( saveTask.ProcessingStatus == EProcessingStatus::Added ) { + if (saveTask.OpType == EOperationType::Add ) { + TString& name = saveTask.Name; + auto search = LastObjs_.find(name); + ui32 foundObjId = 0; + if ( search != LastObjs_.end() ) { + foundObjId = search->second; + } + + if (foundObjId <= saveTask.ObjId ) { + saveTask.ProcessingStatus = EProcessingStatus::Processing; + res = true; + break; + } else { + saveTask.ProcessingStatus = EProcessingStatus::Deleted; + } + } + + if (saveTask.OpType == EOperationType::StreamBufAdd ) { + saveTask.ProcessingStatus = EProcessingStatus::Processing; + res = true; + break; + + } + + } + + } + return res; +} + +bool TNamespaceCache::FindNextTaskInLoadQueue(ui32& taskPos) { + + bool res = false; + for ( taskPos = 0; taskPos < ToLoad_.size(); taskPos++) { + TLoadTask& loadTask = ToLoad_[taskPos]; + if ( loadTask.ProcessingStatus == EProcessingStatus::Added ) { + if (loadTask.OpType == EOperationType::Add ) { + TString& name = loadTask.Name; + auto search = LastObjs_.find(name); + ui32 foundObjId = 0; + if ( search != LastObjs_.end() ) { + foundObjId = search->second; + } + + if (foundObjId > loadTask.ObjId ) { + loadTask.ObjId = foundObjId; + } + loadTask.ProcessingStatus = EProcessingStatus::Processing; + res = true; + break; + } + } + } + return res; +} + +bool TNamespaceCache::FindNextTaskInDeleteQueue(ui32& taskPos) { + + bool res = false; + for ( taskPos = 0; taskPos < ToDelete_.size(); taskPos++) { + TBaseTask& deleteTask = ToDelete_[taskPos]; + if ( deleteTask.ProcessingStatus == EProcessingStatus::Added ) { + deleteTask.ProcessingStatus = EProcessingStatus::Processing; + res = true; + break; + } + } + return res; +} + +void TNamespaceCache::ChangeLastObjId(TString& objName, ui32 prevObjId, ui32 nextObjId) { + auto search = LastObjs_.find(objName); + if ( search != LastObjs_.end() ) { + if (search->second <= prevObjId) { + search->second = nextObjId; + } + } + +} + +void TNamespaceCache::ProcessSaveQueue() { + + ui32 taskPos = 0; + bool saveTaskFound = false; + bool enoughSpaceToWrite = false; + + try { + ToSaveLock_.lock(); + + RemoveCompletedFromSaveQueue(); + + saveTaskFound = FindNextTaskInSaveQueue(taskPos); + + if ( saveTaskFound ) { + + TAtomicSharedPtr<ISpillFile> currSpillMetaFile = CurrSpillMetaFile_; + TAtomicSharedPtr<ISpillFile> currSpillDataFile = CurrSpillDataFile_; + + TSaveTask& saveTask = ToSave_[taskPos]; + ui32 size = saveTask.Buf->Size(); + ui64 offset = currSpillDataFile->Reserve(size); + ui64 total = offset + size; + char * data = saveTask.Buf->Data(); + ui32 taskObjId = saveTask.ObjId; + ui32 prevObjId = taskObjId; + TSpillMetaRecord mr{EOperationType::Add, saveTask.Name, offset, taskObjId, size, 0 }; + ui32 metaSize = mr.Size(); + ui64 metaOffset = currSpillMetaFile->Reserve(metaSize); + + if (total >= std::numeric_limits<ui32>::max() ) { + saveTask.ProcessingStatus = EProcessingStatus::Added; + NextNamespaceFile(false); + ToSaveLock_.unlock(); + } else { + enoughSpaceToWrite = true; + saveTask.ProcessingStatus = EProcessingStatus::Processing; + bool changeObjId = ( !(taskObjId > CurrSpillFileId_ * (1<<16) && (taskObjId < (CurrSpillFileId_ + 1) * (1<<16)) )); + if (changeObjId) { + AdvanceObjId(taskObjId); + mr.SetObjId(taskObjId); + } + ToSaveLock_.unlock(); + XXH32_hash_t hash = XXH32( data, size, 0); + mr.SetDataHash(hash); + + + TBuffer mrBuf; + mr.Pack(mrBuf); + + currSpillDataFile->Write(offset, data, size); + currSpillMetaFile->Write(metaOffset, mrBuf.Data(), metaSize); + + + ToSaveLock_.lock(); + saveTask.ProcessingStatus = EProcessingStatus::Deleted; + if (changeObjId) { + ChangeLastObjId(saveTask.Name, prevObjId, taskObjId); + } + SessionDataSpilled_[saveTask.SessionId] += size; + ToSaveLock_.unlock(); + + with_lock(FilesLock_) { + auto & metaAttributes = SpillMetaFiles_[CurrSpillFileId_]; + metaAttributes->MetaRecords->insert({taskObjId, mr}); + } + } + + } else { + ToSaveLock_.unlock(); + } + + + } catch(...) { + ToSaveLock_.unlock(); + throw; + } + +} + +void TNamespaceCache::ProcessLoadQueue() { + + ui32 taskPos = 0; + bool loadTaskFound = false; + + with_lock(ToLoadLock_) { + RemoveCompletedFromLoadQueue(); + loadTaskFound = FindNextTaskInLoadQueue(taskPos); + } + + if (!loadTaskFound) + return; + + TLoadTask& loadTask = ToLoad_[taskPos]; + TLoadOperationResults lr; + ui32 fileInd = FindFileInd(loadTask.ObjId); + TSpillMetaRecord mr; + TAtomicSharedPtr<TMetaFileAttributes> fileMet; + bool found = false; + + with_lock(FilesLock_) { + found = FindMetaRecordInFile(fileInd, loadTask.ObjId, mr, fileMet ); + } + + + if (!found) + return; + + TBuffer buf(mr.DataSize()); + fileMet->DataFile->Read(mr.Offset(), buf.Data(), mr.DataSize()); + XXH32_hash_t hash = XXH32( buf.Data(), mr.DataSize(), 0); + lr.Buf = MakeAtomicShared<TBuffer>(std::move(buf)); + lr.Status = EOperationStatus::Success; + + if ( hash != mr.DataHash() ) { + lr.Status = EOperationStatus::ChecksumInvalid; + YQL_LOG(ERROR) << "Wrong hash!!!: " << "Buf hash: " << hash << " Meta hash: " << mr.DataHash() << Endl; + } + + SessionDataLoadedFromStorage_[loadTask.SessionId] += mr.DataSize(); + + + loadTask.Promise.SetValue(std::move(lr)); + + +} + +void TNamespaceCache::ProcessDeleteQueue() { + + ui32 taskPos = 0; + bool deleteTaskFound = false; + + with_lock(ToDeleteLock_) { + RemoveCompletedFromDeleteQueue(); + deleteTaskFound = FindNextTaskInDeleteQueue(taskPos); + } + + if (!deleteTaskFound) + return; + + TBaseTask& deleteTask = ToDelete_[taskPos]; + ui32 sessionId = deleteTask.SessionId; + + if (deleteTask.OpType == EOperationType::SessionDelete ) { + with_lock(ToSaveLock_) { + Sessions_.erase(sessionId); + for (auto& t : ToSave_ ) { + if ( t.SessionId == sessionId ) { + t.ProcessingStatus = EProcessingStatus::Deleted; + t.Buf = nullptr; + } + } + } + + with_lock(ToLoadLock_) { + for (auto& t : ToLoad_ ) { + if ( t.SessionId == sessionId ) { + t.ProcessingStatus = EProcessingStatus::Deleted; + t.Buf = nullptr; + } + } + } + } + + + +} + + +void TNamespaceCache::RemoveCompletedFromSaveQueue() { + + for (auto it = ToSave_.begin(); it != ToSave_.end(); ) { + if (it->ProcessingStatus == EProcessingStatus::Deleted) { + ui64 sessionId = it->SessionId; + it = ToSave_.erase(it); + } else { + break; + } + } + +} + +void TNamespaceCache::RemoveCompletedFromLoadQueue() { + + for (auto it = ToLoad_.begin(); it != ToLoad_.end(); ) { + if (it->ProcessingStatus == EProcessingStatus::Deleted) { + ui64 sessionId = it->SessionId; + it = ToLoad_.erase(it); + } else { + break; + } + } + +} + + +void TNamespaceCache::RemoveCompletedFromDeleteQueue() { + + for (auto it = ToDelete_.begin(); it != ToDelete_.end(); ) { + if (it->ProcessingStatus == EProcessingStatus::Deleted) { + ui64 sessionId = it->SessionId; + it = ToDelete_.erase(it); + } else { + break; + } + } + +} + + +bool TNamespaceCache::FindMetaRecordInFile(ui32 fileId, ui32 objId, TSpillMetaRecord& mr, TAtomicSharedPtr<TMetaFileAttributes>& fileMet) { + bool res = false; + TAtomicSharedPtr<TMetaFileAttributes> met = InitMetaFile(fileId); + auto it = met->MetaRecords->find(objId); + if (it != met->MetaRecords->end() ) { + res = true; + mr = it->second; + fileMet = met; + } + + return res; +} + +TAtomicSharedPtr< std::vector<TSpillMetaRecord> > ReadAllRecordsFromMetaFile( TAtomicSharedPtr<ISpillFile> file) { + TAtomicSharedPtr< std::vector<TSpillMetaRecord> > res = MakeAtomicShared<std::vector<TSpillMetaRecord>>(); + file->Seek(0); + TBuffer readbuf(ReadBufSize); + i32 readRes = file->Read(0, readbuf.Data(), ReadBufSize); + TSpillMetaRecord mr; + mr.Unpack(readbuf); + return res; +} + + + TAtomicSharedPtr<TMetaFileAttributes> TNamespaceCache::InitMetaFile(ui32 fileId){ + TAtomicSharedPtr<TMetaFileAttributes> res; + + auto found = SpillMetaFiles_.find(fileId); + if (found != SpillMetaFiles_.end()) { + res = found->second; + } else { + THolder<ISpillFile> metFile = StorageI_->CreateSpillFile(Name_, TString("ydbspl.") + std::to_string(fileId) + TString(".0.met"), MetaFileIncreaseStep); + THolder<ISpillFile> datFile = StorageI_->CreateSpillFile(Name_, TString("ydbspl.") + std::to_string(fileId) + TString(".0.dat"), DataFileIncreaseStep); + res = MakeAtomicShared<TMetaFileAttributes>(); + res->MetaFile = std::move( metFile ); + res->DataFile = std::move(datFile); + SpillMetaFiles_[fileId] = res; + } + + + + return res; +} + + +TAtomicSharedPtr<ISpillFile> FindSpillFile(ui32 objId, EFileType fileType = EFileType::Meta) { + TAtomicSharedPtr<ISpillFile> res; + return res; +} + +TNamespaceCache::TNamespaceCache(const TString& name, ui32 id, TAtomicSharedPtr<ISpillStorage> storageI) : + Name_(name), + Id_(id), + StorageI_(storageI) { + + }; + + +} +} diff --git a/ydb/library/yql/core/spilling/namespace_cache.h b/ydb/library/yql/core/spilling/namespace_cache.h new file mode 100644 index 00000000000..2e648869d89 --- /dev/null +++ b/ydb/library/yql/core/spilling/namespace_cache.h @@ -0,0 +1,215 @@ +#include "interface/spilling.h" +#include "storage/storage.h" + +#include <map> +#include <deque> +#include <set> +#include <unordered_map> + + +namespace NYql { +namespace NSpilling { + + + +const ui32 MetaFileIncreaseStep = 10000; // 10 KB increase in meta file size +const ui32 DataFileIncreaseStep = 100000000; // 100 MB increase in data file size +const ui32 ReadBufSize = 4 * 1024 * 1024; // 4 MB read buffer size + +// Defines possible processing states of the task +enum class EProcessingStatus { + Added = 1, // Task added to queue + Processing = 2, // Task is processing + Completed = 3, // Task is completed + NeedRetry = 4, // Scheduled for retry + Failed = 5, // Task is failed + Deleted = 6, // Task is deleted, no valid internal state + NewVersion = 7 // New version of object is available +}; + + +// Base task for namespace operations +struct TBaseTask { + EOperationType OpType = EOperationType::Add; // Type of operation + ui32 ObjId; // Identifier of the object + ui32 SessionId; // Identifier of the session for this task + TString Name; // Name of the object or stream + ui32 StreamBufId; // Id of the object inside the stream (0, 1, 2, ...) + TAtomicSharedPtr<TBuffer> Buf; // Data to save or load + EProcessingStatus ProcessingStatus = EProcessingStatus::Added; // Contains current status of task processing stage +}; + + +// Task for save operations +struct TSaveTask : public TBaseTask { + NThreading::TPromise<TOperationResults> Promise; // Promise to report operation results +}; + + +// Task for load operations +struct TLoadTask : public TBaseTask { + NThreading::TPromise<TLoadOperationResults> Promise; // Promise to report load operation results +}; + +// Defines possible retention period of file. All temporary files are deleted automacially on session close. +// Persistent files store objects as defined in object retention period. +enum class EFileLifespan { + Temporary = 0, + Persistent = 1 +}; + +// Defines possible types of spill files. Meta files contain small index of objects stored in data files +enum class EFileType { + Meta = 0, + Data = 1 +}; + +// Namespace file description +struct TNamespaceFileDesc { + TString Name; // Full name of the file + EFileType Type = EFileType::Meta; // Type of the file + EFileLifespan Lifespan = EFileLifespan::Temporary; // Lifespan of the spill file. All unlocked temporary files are deleted automatically + ui32 Id = 1; // Serial id of the file in the namespace. Id of the file defines interval of Object Ids, stored in file + ui32 Version = 1; // Version of the file in the namespace for the same id. + +}; + +// Session info stored in the namespace cache +struct TSessionDesc { + ui32 MinObjId; // Mininum object id associated with session +}; + +// Attributes of meta file to be used +struct TMetaFileAttributes { + TAdaptiveLock Lock; // Lock to serialize access to file attributes + bool Valid; // True if namespace meta file exists and valid + TString Name; // Name of the file + ui32 Id = 0; // Id of the file in the namespace + TAtomicSharedPtr<ISpillFile> MetaFile; // File handle of the meta file + TAtomicSharedPtr<ISpillFile> DataFile; // File handle of the data file + ui32 StartPos = 0; // Start valid position in file + ui32 EndPos = 0; // End of valid position in file + ui32 FirstObjId = 0; // First object id in file + ui32 LastObjId = 0; // Last object id in file + TAtomicSharedPtr< std::map<ui32, TSpillMetaRecord> > MetaRecords; // Cached meta records of the file +}; + + + +// Class to represent internal cache for particular namespace +class TNamespaceCache { +public: + // Adds required spilling data to save queue + NThreading::TFuture<TOperationResults> Save(const TString& objName, TBuffer && buf, ui32 sessionId, bool isStream = false ); + + // Loads data from namespace cache or long term storage + NThreading::TFuture<TLoadOperationResults> Load(const TString& name, ui32 sessionId, EObjectsLifetime objLifetime = EObjectsLifetime::DeleteAfterLoad, bool isStream = false, ui32 streamId = 0 ); + + // Dispatches worker thread to process queues + void Dispatch(); + + // Return data statistics for particular session + TSessionDataStat GetSessionStat(ui32 sessionId); + + // Closes session and clears all resources + void CloseSession(ui32 sessionId); + + // Makes garbage collection for files stored in namespace + void GarbageCollection(); + + // Returns current number of objects stored in particular stream name + ui64 StreamSize(const TString& name); + + // name - name of the namespace, id - internal id, storageI - storage interface to save Namespace Cache state + TNamespaceCache(const TString& name, ui32 id, TAtomicSharedPtr<ISpillStorage> storageI); +private: + + // Creates next namespace file. If openExisting = true, last namespace file is reused + void NextNamespaceFile(bool openExisting = true); + + // Advances next object id, taking into account cyclic requirement + inline void AdvanceObjId(ui32& objId); + + // Finds object with ObjId in Save queue. Returns true if found and object position in queue + inline bool FindObjInSaveQueue(ui32 objId, ui32& pos); + + // Updates mininum object id for particular session if required + inline void UpdateMinSessionId(ui32 sessionId, ui32 objId); + + // Deletes task from save queue. Returns bytes deleted in save task buffer. + ui32 DeleteTaskFromSaveQueue(ui32 objId, EProcessingStatus reason = EProcessingStatus::Deleted); + + // Changes object id for the object + void ChangeLastObjId(TString& objName, ui32 prevObjId, ui32 nextObjId); + + // Processes save queue + void ProcessSaveQueue(); + + // Processes load queue + void ProcessLoadQueue(); + + // Processes delete queue + void ProcessDeleteQueue(); + + // Finds next valid save task in save queue and returns true is task found. Task position in queue is returned in taskPos. + bool FindNextTaskInSaveQueue(ui32& taskPos); + + // Finds next valid task in load queue and returns true is task found. Task position in queue is returned in taskPos. + bool FindNextTaskInLoadQueue(ui32& taskPos); + + // Finds next valid task in delete queue and returns true is task found. Task position in queue is returned in taskPos. + bool FindNextTaskInDeleteQueue(ui32& taskPos); + + // Removes all completed tasks from beginning of ToSave_ queue + void RemoveCompletedFromSaveQueue(); + + // Removes all completed tasks from beginning of ToLoad_ queue + void RemoveCompletedFromLoadQueue(); + + // Removes all completed tasks from beginning of ToDelete_ queue + void RemoveCompletedFromDeleteQueue(); + + // Returns true if object is found in meta file fileId. + // Meta record for particular object is returned in mr argument. File meta information is returned in fileMet record. + bool FindMetaRecordInFile(ui32 fileId, ui32 objId, TSpillMetaRecord& mr, TAtomicSharedPtr<TMetaFileAttributes>& fileMet ); + + // Inits meta file with file id + TAtomicSharedPtr<TMetaFileAttributes> InitMetaFile(ui32 fileId); + + // Marks obj id as deleted + inline void SetDeletedBit(ui32 objId); + + // Returns required file handle for particular object id + TAtomicSharedPtr<ISpillFile> FindSpillFile(ui32 objId, EFileType fileType = EFileType::Meta); + + // Finds iterator of object with provided objId inside + + TAdaptiveLock FilesLock_; // Lock to serialize access to internal data structures associated with files content + TAdaptiveLock ToSaveLock_; // Lock to serialize access to save queue + TAdaptiveLock ToLoadLock_; // Lock to serialize access to load queue + TAdaptiveLock ToDeleteLock_; // Lock to serialize access to delete queue + TString Name_; // Name of the namespace + ui32 Id_ = 0; // Internal id of the namespace + std::deque<TSaveTask> ToSave_; // Queue to spill data + std::deque<TLoadTask> ToLoad_; // Queue to load spilled data + std::deque<TBaseTask> ToDelete_; // Queue to delete spilled data + std::unordered_map<TString, ui32> LastObjs_; // Latest version of objects with particular name + std::unordered_map<TString, std::vector<ui32>> LastStreamIds_; // Latest buff id of particular stream + std::unordered_map<ui32, TSessionDesc> Sessions_; // Sessions descriptions by session id key + std::unordered_map<ui32, std::atomic<ui64>> SessionDataProvided_; // Data provided to spill per session + std::unordered_map<ui32, std::atomic<ui64>> SessionDataSpilled_; // Data spilled to storage from memory per session + std::unordered_map<ui32, std::atomic<ui64>> SessionDataLoadedFromStorage_; // Data loaded from storage to memory per session + std::unordered_map<ui32, std::atomic<ui64>> SessionDataLoadedFromMemory_; // Data loaded from memory to memory per session + TAtomicSharedPtr<ISpillStorage> StorageI_; // Interface to storage to work with namespace + TAtomicSharedPtr<ISpillFile> CurrSpillMetaFile_ = nullptr; // Current spill meta file to write data + TAtomicSharedPtr<ISpillFile> CurrSpillDataFile_ = nullptr; // Current spill file to write data + ui32 CurrSpillFileId_ = 0; // Current id of spill file in the namespace + std::atomic<ui32> NextObjId_ = 0; // Next object id in the namespace + std::map<ui32, TAtomicSharedPtr<TMetaFileAttributes> > SpillMetaFiles_; // Current open spill meta files for particular namespace + std::set<ui32> SpillFilesIdToDelete_; // Id of spill files which need to be deleted + +}; + + +} +} diff --git a/ydb/library/yql/core/spilling/namespaces_list.cpp b/ydb/library/yql/core/spilling/namespaces_list.cpp new file mode 100644 index 00000000000..977fd9dbefe --- /dev/null +++ b/ydb/library/yql/core/spilling/namespaces_list.cpp @@ -0,0 +1,167 @@ +#include "namespaces_list.h" +#include "interface/spilling.h" +#include "storage/storage.h" + + +#include <map> + + +namespace NYql { +namespace NSpilling { + + +TAtomicSharedPtr<TNamespaceCache> TNamespacesList::GetNamespaceCache(const TString& name){ + TAtomicSharedPtr<TNamespaceCache> res; + with_lock(NamespacesMapLock_) { + if ( auto found = NamespacesMap_.find(name); found != NamespacesMap_.end() ) { + res = found->second; + } else { + CurrNamespaceId_++; + res = MakeAtomicShared<TNamespaceCache>(name, CurrNamespaceId_, StorageI_); + NamespacesMap_.emplace(name, res); + } + } + return res; +} + + +void TNamespacesList::PushNamespaceToProcess(const TString& name, ui32 sessionId) { + with_lock(QueueLock_) { + NamespacesQueue_.push_back(name); + if (ActiveWorkerThreads_ < NamespacesQueue_.size()) { + WorkingSemaphore_.release(); + } + } + with_lock(SessionsLock_) { + SessionNamespaces_[sessionId].emplace(name); + } +} + +bool TNamespacesList::PopNamespaceToProcess(TString& name) { + bool res = false; + with_lock(QueueLock_) { + if ( NamespacesQueue_.size() > 0) { + name = NamespacesQueue_.front(); + NamespacesQueue_.pop_front(); + res = true; + } else { + res = false; + } + } + return res; +} + +bool TNamespacesList::WaitForTask() { + std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); + auto diff = std::chrono::milliseconds(1000); + ActiveWorkerThreads_--; + bool task = WorkingSemaphore_.try_acquire_for(diff); + ActiveWorkerThreads_++; + return task; +} + +void TNamespacesList::PerformGarbageCollection() { + + with_lock(GarbageCollectionLock_) { + if ( NamespacesForGarbageCollection_.size() == 0 ) { + NamespacesForGarbageCollection_ = StorageI_->GetNamespaces(); + } + } + + bool needGarbageCollection = true; + + if (GarbageCollectionThreads_ <= MaxGarbageCollectionThreads ) { + GarbageCollectionThreads_++; + } else { + return; + } + + while (needGarbageCollection) { + TString ns; + with_lock( GarbageCollectionLock_ ) { + if ( NamespacesForGarbageCollection_.size() > 0 ) { + ns = NamespacesForGarbageCollection_.back(); + NamespacesForGarbageCollection_.pop_back(); + } else { + needGarbageCollection = false; + break; + } + } + + TAtomicSharedPtr<TNamespaceCache> nsc; + nsc = GetNamespaceCache(ns); + nsc->GarbageCollection(); + + } + + GarbageCollectionThreads_--; + + +} + +void TNamespacesList::RemoveNamespaceCache(const TString& name){ + with_lock(NamespacesMapLock_) { + NamespacesMap_.erase(name); + } +} + +bool TNamespacesList::CheckForStop() { + bool res = false; + with_lock(NamespacesMapLock_) { + res = StopFlag_; + } + return res; +} + +void TNamespacesList::Stop() { + with_lock(NamespacesMapLock_) { + StopFlag_ = true; + for (ui32 i = 0; i < MaxThreadPoolSize; i++ ) { + WorkingSemaphore_.release(); + } + } +} + +TSessionDataStat TNamespacesList::GetSessionDataStat(ui32 sessionId) { + TSessionDataStat res; + with_lock(SessionsLock_) { + auto found = SessionNamespaces_.find(sessionId); + if (found != SessionNamespaces_.end()) { + for (auto& it: found->second) { + TSessionDataStat ns_res; + auto nsc = GetNamespaceCache(it); + ns_res = nsc->GetSessionStat(sessionId); + res.Provided += ns_res.Provided; + res.InMemory += ns_res.InMemory; + res.Spilled += ns_res.Spilled; + res.LoadedFromStorage += ns_res.LoadedFromStorage; + res.LoadedFromMemory += ns_res.LoadedFromMemory; + } + } + } + return res; +} + +void TNamespacesList::CloseSession(ui32 sessionId) { + with_lock(SessionsLock_) { + auto found = SessionNamespaces_.find(sessionId); + if (found != SessionNamespaces_.end()) { + for (auto& it: found->second) { + TSessionDataStat ns_res; + auto nsc = GetNamespaceCache(it); + nsc->CloseSession(sessionId); + } + } + } + + +} + +TNamespacesList::TNamespacesList(ui32 threadPoolSize, TAtomicSharedPtr<ISpillStorage> storageI) : + ActiveWorkerThreads_(threadPoolSize), + StorageI_(storageI) { + LastGarbageCollection_ = std::chrono::steady_clock::now(); +} + +} +} diff --git a/ydb/library/yql/core/spilling/namespaces_list.h b/ydb/library/yql/core/spilling/namespaces_list.h new file mode 100644 index 00000000000..e1b3fb6f0fd --- /dev/null +++ b/ydb/library/yql/core/spilling/namespaces_list.h @@ -0,0 +1,54 @@ +#include "interface/spilling.h" +#include "storage/storage.h" +#include "namespace_cache.h" + +#include <map> +#include <deque> +#include <semaphore> +#include <set> + + +namespace NYql { +namespace NSpilling { + +const ui32 DefaultThreadPoolSize = 10; +const ui32 MaxThreadPoolSize = 30; + +const ui32 GarbageCollectionPeriodMs = 30000; // Default timeout for garbage collection activity (milliseconds) +const ui32 MaxGarbageCollectionThreads = 3; // Maximum number of threads to perform garbage collection + +class TNamespacesList { +public: + TAtomicSharedPtr<TNamespaceCache> GetNamespaceCache(const TString& name); + void PushNamespaceToProcess(const TString& name, ui32 sessionId); // Adds particular namespace for async processing + bool PopNamespaceToProcess(TString& name); // Returns true if there is next namespace for processing. Namespace is returned in TString& name argument. + bool WaitForTask(); // Waits for the next task for worker thread + void PerformGarbageCollection(); // Performs garbage collection with GarbageCollectionPeriodMs + void RemoveNamespaceCache(const TString& name); + bool CheckForStop(); // Checks if worker thread need to be stopped + void Stop(); // Signal to stop workers threads + TSessionDataStat GetSessionDataStat(ui32 SessionId); // Returns statistics for data spilled and loaded during session + void CloseSession(ui32 SessionId); // Closes session and clears all resources + TNamespacesList(ui32 threadPoolSize, TAtomicSharedPtr<ISpillStorage> storageI); +private: + TAdaptiveLock NamespacesMapLock_; // Lock to serialize access to internal map data structures + TAdaptiveLock QueueLock_; // Lock for NamespacesQueue + TAdaptiveLock SessionsLock_; // Lock for SessionNamespaces lock + TAdaptiveLock GarbageCollectionLock_; // Lock for Garbage collection activity + std::atomic<ui32> ActiveWorkerThreads_ = 0; // Number of current active working threads + std::atomic<ui32> GarbageCollectionThreads_ = 0; // Number of current active working threads + std::counting_semaphore<> WorkingSemaphore_{MaxThreadPoolSize}; // Semaphore to wait for the worker threads activation + ui32 CurrNamespaceId_ = 1; // Current namespace id to assign to new namespace + std::map<TString, TAtomicSharedPtr<TNamespaceCache>> NamespacesMap_; // Map to find required namespace cache by name + std::map<ui32, std::set<TString> > SessionNamespaces_; // Set of all namespace names for particular session + std::deque<TString> NamespacesQueue_; // Queue to process namespaces with pending tasks by thread pool + bool StopFlag_ = false; + TAtomicSharedPtr<ISpillStorage> StorageI_; // Interface to storage to work with namespace + std::chrono::steady_clock::time_point LastGarbageCollection_; // Last time when namespace garbage collection was performed + std::vector<TString> NamespacesForGarbageCollection_; // List of namespaces to perform garbage collection + +}; + + +} +} diff --git a/ydb/library/yql/core/spilling/spilling_imp.cpp b/ydb/library/yql/core/spilling/spilling_imp.cpp new file mode 100644 index 00000000000..f6b5f74c9ff --- /dev/null +++ b/ydb/library/yql/core/spilling/spilling_imp.cpp @@ -0,0 +1,245 @@ +#include "spilling_imp.h" +#include "interface/spilling.h" +#include "storage/storage.h" + +#include <ydb/library/yql/utils/log/log.h> + +#include <random> +#include <util/folder/path.h> +#include <filesystem> +#include <iostream> +#include <limits> +#include <unordered_map> +#include <chrono> + +#include <util/string/split.h> + +#include <util/folder/dirut.h> +#include <util/generic/algorithm.h> +#include <util/generic/vector.h> +#include <util/generic/yexception.h> +#include <util/generic/ptr.h> +#include <util/generic/utility.h> +#include <util/system/file.h> +#include <util/system/file_lock.h> +#include <util/system/fs.h> +#include <util/system/maxlen.h> +#include <util/system/mutex.h> +#include <util/system/utime.h> +#include <util/system/thread.h> + + + +namespace NYql { +namespace NSpilling { + +THolder<ISession> TTempStorageProxyImp::CreateSession() { + if (CurrSessId_ == std::numeric_limits<ui32>::max() ) { + CurrSessId_ = 1; + } + ui32 sessId = CurrSessId_.fetch_add(1); + return MakeHolder<TSessionImp>(sessId, NsList_, StorageI_); + +} + +THolder<IObjectsIterator> TTempStorageProxyImp::CreateIterator( + const TMaybe<TString>& objNamespace, + const TMaybe<TString>& objName, + bool onlyValid) { + + return MakeHolder<TStorageIteratorImp>(objNamespace, objName, onlyValid); +} + +TTempStorageExecutionPolicy TTempStorageProxyImp::ExecutionPolicy() { + return Policy_; +} + +TOperationResults TTempStorageProxyImp::SetExecutionPolicy(const TTempStorageExecutionPolicy & policy) { + Policy_ = policy; +} + +NThreading::TFuture<TOperationResults> TTempStorageProxyImp::Delete(const TString & objNamespace, const TMaybe<TString> & name) { + NThreading::TFuture<TOperationResults> res; + return res; +} + + +TOperationResults TTempStorageProxyImp::LastOperationResults(){ + return OperationResults_; +} + +ui32 Ui32Rand(const ui32 & min, const ui32 & max) { + static thread_local std::mt19937 generator; + std::uniform_int_distribution<ui32> distribution(min,max); + return distribution(generator); +} + +// Worker function in threads pool +void ProcessThreadPoolTasks(TAtomicSharedPtr<TNamespacesList> nsl, TAtomicSharedPtr<ISpillStorage> sti){ + bool stopFlag = false; + bool haveNamespaceToProcess = false; + TString namespaceToProcess; + ui32 periodShift = Ui32Rand(0, GarbageCollectionPeriodMs / 8); + + std::chrono::steady_clock::time_point lastGarbageCollection = std::chrono::steady_clock::now(); + + TThread::TId this_id = TThread::CurrentThreadNumericId(); + + while( !stopFlag ) { + try { + bool task = nsl->WaitForTask(); + + std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now(); + ui64 period = std::chrono::duration_cast<std::chrono::milliseconds>(now - lastGarbageCollection).count(); + if ( period > GarbageCollectionPeriodMs + periodShift ) { + + if ( this_id % DefaultThreadPoolSize <= MaxGarbageCollectionThreads) { + lastGarbageCollection = now; + nsl->PerformGarbageCollection(); + now = std::chrono::steady_clock::now(); + ui64 execTime = std::chrono::duration_cast<std::chrono::microseconds>(now - lastGarbageCollection).count(); + YQL_LOG(INFO) << "Thread " << this_id << " garbage collection (microseconds): " << execTime << Endl; + } + } + + haveNamespaceToProcess = nsl->PopNamespaceToProcess(namespaceToProcess); + if (haveNamespaceToProcess) { + TAtomicSharedPtr<TNamespaceCache> nsc = nsl->GetNamespaceCache(namespaceToProcess); + nsc->Dispatch(); + } else { + } + stopFlag = nsl->CheckForStop(); + } catch(...) { + + YQL_LOG(ERROR) << "Error happened..." << FormatCurrentException() << Endl; + } + } + YQL_LOG(INFO) << "Stopping spilling thread " << this_id << " ..." << Endl; +} + + +TTempStorageProxyImp::TTempStorageProxyImp(const TFileStorageConfig & config, const TTempStorageExecutionPolicy & policy, THolder<ISpillStorage>&& storage) : + StorageConfig_(config), + Policy_(policy), + StorageI_(std::move(storage)) +{ + OperationResults_.Status = EOperationStatus::Success; + NsList_ = MakeAtomicShared<TNamespacesList>(DefaultThreadPoolSize, StorageI_); + for (ui32 i = 0; i < DefaultThreadPoolSize; i++) { + ThreadPool_.push_back(std::thread(ProcessThreadPoolTasks, NsList_, StorageI_)); + } +} + + +TTempStorageProxyImp::~TTempStorageProxyImp(){ + YQL_LOG(INFO) << "Stopping nslist ..." << Endl; + NsList_->Stop(); + YQL_LOG(INFO) << "Stopping threads ..." << Endl; + for(auto & t: ThreadPool_) { + t.join(); + } +} + + + +NThreading::TFuture<TOperationResults> TSessionImp::Save(const TString & objNamespace, const TString & name, TBuffer && buf){ + TAtomicSharedPtr<TNamespaceCache> nsc = NsList_->GetNamespaceCache(objNamespace); + NsList_->PushNamespaceToProcess(objNamespace, SessionId_); + return nsc->Save(name, std::move(buf), SessionId_); +} + +NThreading::TFuture<TLoadOperationResults> TSessionImp::Load(const TString & objNamespace, const TString & name, EObjectsLifetime objLifetime ){ + TAtomicSharedPtr<TNamespaceCache> nsc = NsList_->GetNamespaceCache(objNamespace); + NsList_->PushNamespaceToProcess(objNamespace, SessionId_); + return nsc->Load(name, SessionId_, objLifetime); +} + + +TSessionDataStat TSessionImp::GetSessionDataStat(){ + return NsList_->GetSessionDataStat(SessionId_); +} + +TSessionExecutionPolicy TSessionImp::ExecutionPolicy(){ + return Policy_; +} + + +TOperationResults TSessionImp::SetExecutionPolicy(const TSessionExecutionPolicy& policy){ + TOperationResults res; + Policy_ = policy; + return res; +} + +std::pair<THolder<IStream>, TOperationResults> TSessionImp::OpenStream(const TString& objNamespace, const TString& streamName ) { + THolder<TStreamImp> sti = MakeHolder<TStreamImp>(objNamespace, streamName, NsList_, SessionId_); + TOperationResults res; + return std::make_pair<THolder<IStream>, TOperationResults> (std::move(sti), std::move(res)); +} + + +TSessionImp::TSessionImp(ui32 sessionId, TAtomicSharedPtr<TNamespacesList> nsList, TAtomicSharedPtr<ISpillStorage> storage) : + SessionId_(sessionId), + NsList_(nsList), + StorageI_(storage) { + +} + +TStreamImp::TStreamImp(const TString& objNamespace, const TString& streamName, TAtomicSharedPtr<TNamespacesList> nsList, ui32 sessionId ) : + Namespace_(objNamespace), + Name_(streamName), + NsList_(nsList), + SessionId_(sessionId) {}; + +TSessionImp::~TSessionImp(){ + NsList_->CloseSession(SessionId_); +} + + + +bool TStorageIteratorImp::Next(TTempObjectDesc & object) { + return false; +} + +TStorageIteratorImp::TStorageIteratorImp(const TMaybe<TString>& objNamespace, const TMaybe<TString>& objName, bool onlyValid) { +} + + +NThreading::TFuture<TOperationResults> TStreamImp::Save(TBuffer&& buf){ + TAtomicSharedPtr<TNamespaceCache> nsc = NsList_->GetNamespaceCache(Namespace_); + NsList_->PushNamespaceToProcess(Namespace_, SessionId_); + return nsc->Save(Name_, std::move(buf), SessionId_, true); +} + +ui64 TStreamImp::Size() { + TAtomicSharedPtr<TNamespaceCache> nsc = NsList_->GetNamespaceCache(Namespace_); + return nsc->StreamSize(Name_); + +} + +NThreading::TFuture<TLoadOperationResults> TStreamImp::Load(ui64 bufferId, EObjectsLifetime objLifetime ) { + TAtomicSharedPtr<TNamespaceCache> nsc = NsList_->GetNamespaceCache(Namespace_); + NsList_->PushNamespaceToProcess(Namespace_, SessionId_); + return nsc->Load(Name_, SessionId_, objLifetime, true, bufferId); + + +} + +NThreading::TFuture<TOperationResults> TStreamImp::Close() { + NThreading::TFuture<TOperationResults> res; + return res; +} + + + + +std::pair< THolder<ITempStorageProxy>, TOperationResults > CreateFileStorageProxy(const TFileStorageConfig & config, const TTempStorageExecutionPolicy & policy ) { + + std::pair< THolder<ISpillStorage>, TOperationResults > sps = OpenFileStorageForSpilling(config); + THolder<TTempStorageProxyImp> sp = MakeHolder<TTempStorageProxyImp>(config, policy, std::move(sps.first)); + TOperationResults res = sps.second; + return std::make_pair< THolder<ITempStorageProxy>, TOperationResults >( std::move(sp), std::move(res) ); +} + + +} +}
\ No newline at end of file diff --git a/ydb/library/yql/core/spilling/spilling_imp.h b/ydb/library/yql/core/spilling/spilling_imp.h new file mode 100644 index 00000000000..271163c1564 --- /dev/null +++ b/ydb/library/yql/core/spilling/spilling_imp.h @@ -0,0 +1,98 @@ +#include "interface/spilling.h" +#include "storage/storage.h" +#include "namespaces_list.h" + +#include <thread> +#include <set> + +namespace NYql { +namespace NSpilling { + +const ui64 MagicForFileRecord = 0xA957248FEED9E4CE; + + + + + +class TTempStorageProxyImp : public ITempStorageProxy { +public: + THolder<ISession> CreateSession(); + THolder<IObjectsIterator> CreateIterator( const TMaybe<TString>& objNamespace = TMaybe<TString>(), + const TMaybe<TString>& objName = TMaybe<TString>(), + bool onlyValid = true); + TTempStorageExecutionPolicy ExecutionPolicy(); + TOperationResults SetExecutionPolicy(const TTempStorageExecutionPolicy& policy); + NThreading::TFuture<TOperationResults> Delete(const TString& objNamespace, const TMaybe<TString>& name); + TOperationResults LastOperationResults(); + TTempStorageProxyImp(const TFileStorageConfig & config, const TTempStorageExecutionPolicy & policy, THolder<ISpillStorage>&& storage); + ~TTempStorageProxyImp(); + +private: + TFileStorageConfig StorageConfig_; + TTempStorageExecutionPolicy Policy_; + TAtomicSharedPtr<ISpillStorage> StorageI_; + TOperationResults OperationResults_; // Last operation results + std::atomic<ui32> CurrSessId_ = 1; // Current session id to assign new session + std::vector<std::thread> ThreadPool_; // Thread pool to process spilling tasks + TAtomicSharedPtr<TNamespacesList> NsList_; // List of namespaces to forward requests + +}; + +// Worker function in threads pool +void ProcessThreadPoolTasks(TAtomicSharedPtr<TNamespacesList> nsl, TAtomicSharedPtr<ISpillStorage> sti); + + +// Class to store session object ids +class TSessionIds { + ui32 NsId_ = 0; + ui32 ObjId_ = 0; +}; + +class TSessionImp: public ISession { +public: + NThreading::TFuture<TOperationResults> Save(const TString & objNamespace, const TString & name, TBuffer && buf); + NThreading::TFuture<TLoadOperationResults> Load(const TString & objNamespace, const TString & name, EObjectsLifetime objLifetime = EObjectsLifetime::DeleteAfterLoad ); + TSessionDataStat GetSessionDataStat(); + TSessionExecutionPolicy ExecutionPolicy(); + TOperationResults SetExecutionPolicy(const TSessionExecutionPolicy& policy); + std::pair<THolder<IStream>, TOperationResults> OpenStream(const TString& objNamespace, const TString& streamName ); + TSessionImp(ui32 sessionId, TAtomicSharedPtr<TNamespacesList> nsList, TAtomicSharedPtr<ISpillStorage> storage); + ~TSessionImp(); + +private: + ui32 SessionId_; + TSessionExecutionPolicy Policy_; + TAtomicSharedPtr<TNamespacesList> NsList_; // List of namespaces to forward requests + TAtomicSharedPtr<ISpillStorage> StorageI_; // Storage interface to spill session data +}; + +class TStorageIteratorImp: public IObjectsIterator { +public: + bool Next(TTempObjectDesc & object); + TStorageIteratorImp(const TMaybe<TString>& objNamespace = TMaybe<TString>(), + const TMaybe<TString>& objName = TMaybe<TString>(), + bool onlyValid = true); +private: + bool NameSpacesIterator_ = false; + TVector<TString> ObjNamespaces_; + TVector<TString>::const_iterator CurrNamespace_; +}; + + +class TStreamImp: public IStream { +public: + NThreading::TFuture<TOperationResults> Save(TBuffer&& buf); + ui64 Size(); + NThreading::TFuture<TLoadOperationResults> Load(ui64 bufferId = 0, EObjectsLifetime objLifetime = EObjectsLifetime::DeleteAfterLoad); + NThreading::TFuture<TOperationResults> Close(); + TStreamImp(const TString& objNamespace, const TString& streamName, TAtomicSharedPtr<TNamespacesList> nsList, ui32 sessionId ); +private: + TString Namespace_; + TString Name_; + TAtomicSharedPtr<TNamespacesList> NsList_; + ui32 SessionId_; +}; + + +} +}
\ No newline at end of file diff --git a/ydb/library/yql/core/spilling/storage/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/spilling/storage/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..cd28502e3b3 --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,31 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(file_storage) +add_subdirectory(ut) + +add_library(core-spilling-storage) +target_compile_options(core-spilling-storage PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION + $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything> +) +target_link_libraries(core-spilling-storage PUBLIC + contrib-libs-cxxsupp + yutil + spilling-storage-file_storage + yql-minikql-codegen + llvm12-lib-IR + lib-ExecutionEngine-MCJIT + llvm12-lib-Linker + lib-Target-X86 + Target-X86-AsmParser + lib-Transforms-IPO +) +target_sources(core-spilling-storage PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/storage.cpp +) diff --git a/ydb/library/yql/core/spilling/storage/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/spilling/storage/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..4150b7499b7 --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/CMakeLists.linux-aarch64.txt @@ -0,0 +1,32 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(file_storage) +add_subdirectory(ut) + +add_library(core-spilling-storage) +target_compile_options(core-spilling-storage PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION + $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything> +) +target_link_libraries(core-spilling-storage PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + spilling-storage-file_storage + yql-minikql-codegen + llvm12-lib-IR + lib-ExecutionEngine-MCJIT + llvm12-lib-Linker + lib-Target-X86 + Target-X86-AsmParser + lib-Transforms-IPO +) +target_sources(core-spilling-storage PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/storage.cpp +) diff --git a/ydb/library/yql/core/spilling/storage/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/spilling/storage/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..4150b7499b7 --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/CMakeLists.linux-x86_64.txt @@ -0,0 +1,32 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(file_storage) +add_subdirectory(ut) + +add_library(core-spilling-storage) +target_compile_options(core-spilling-storage PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION + $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything> +) +target_link_libraries(core-spilling-storage PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + spilling-storage-file_storage + yql-minikql-codegen + llvm12-lib-IR + lib-ExecutionEngine-MCJIT + llvm12-lib-Linker + lib-Target-X86 + Target-X86-AsmParser + lib-Transforms-IPO +) +target_sources(core-spilling-storage PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/storage.cpp +) diff --git a/ydb/library/yql/core/spilling/storage/CMakeLists.txt b/ydb/library/yql/core/spilling/storage/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/core/spilling/storage/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/spilling/storage/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..cd28502e3b3 --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/CMakeLists.windows-x86_64.txt @@ -0,0 +1,31 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(file_storage) +add_subdirectory(ut) + +add_library(core-spilling-storage) +target_compile_options(core-spilling-storage PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION + $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything> +) +target_link_libraries(core-spilling-storage PUBLIC + contrib-libs-cxxsupp + yutil + spilling-storage-file_storage + yql-minikql-codegen + llvm12-lib-IR + lib-ExecutionEngine-MCJIT + llvm12-lib-Linker + lib-Target-X86 + Target-X86-AsmParser + lib-Transforms-IPO +) +target_sources(core-spilling-storage PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/storage.cpp +) diff --git a/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..9028583a24d --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,29 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(spilling-storage-file_storage) +target_compile_options(spilling-storage-file_storage PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION + $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything> +) +target_link_libraries(spilling-storage-file_storage PUBLIC + contrib-libs-cxxsupp + yutil + yql-utils-log + yql-minikql-codegen + llvm12-lib-IR + lib-ExecutionEngine-MCJIT + llvm12-lib-Linker + lib-Target-X86 + Target-X86-AsmParser + lib-Transforms-IPO +) +target_sources(spilling-storage-file_storage PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp +) diff --git a/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..e86132790a7 --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.linux-aarch64.txt @@ -0,0 +1,30 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(spilling-storage-file_storage) +target_compile_options(spilling-storage-file_storage PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION + $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything> +) +target_link_libraries(spilling-storage-file_storage PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yql-utils-log + yql-minikql-codegen + llvm12-lib-IR + lib-ExecutionEngine-MCJIT + llvm12-lib-Linker + lib-Target-X86 + Target-X86-AsmParser + lib-Transforms-IPO +) +target_sources(spilling-storage-file_storage PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp +) diff --git a/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..e86132790a7 --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.linux-x86_64.txt @@ -0,0 +1,30 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(spilling-storage-file_storage) +target_compile_options(spilling-storage-file_storage PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION + $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything> +) +target_link_libraries(spilling-storage-file_storage PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + yql-utils-log + yql-minikql-codegen + llvm12-lib-IR + lib-ExecutionEngine-MCJIT + llvm12-lib-Linker + lib-Target-X86 + Target-X86-AsmParser + lib-Transforms-IPO +) +target_sources(spilling-storage-file_storage PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp +) diff --git a/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.txt b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..9028583a24d --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.windows-x86_64.txt @@ -0,0 +1,29 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(spilling-storage-file_storage) +target_compile_options(spilling-storage-file_storage PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION + $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything> +) +target_link_libraries(spilling-storage-file_storage PUBLIC + contrib-libs-cxxsupp + yutil + yql-utils-log + yql-minikql-codegen + llvm12-lib-IR + lib-ExecutionEngine-MCJIT + llvm12-lib-Linker + lib-Target-X86 + Target-X86-AsmParser + lib-Transforms-IPO +) +target_sources(spilling-storage-file_storage PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp +) diff --git a/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp b/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp new file mode 100644 index 00000000000..29c8ab020ea --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp @@ -0,0 +1,260 @@ +#pragma once + +#include <library/cpp/threading/future/async.h> +#include <util/folder/path.h> +#include <filesystem> +#include <iostream> +#include <limits> +#include <unordered_map> +#include <chrono> + +#include <util/string/split.h> + +#include <util/folder/dirut.h> +#include <util/generic/algorithm.h> +#include <util/generic/vector.h> +#include <util/generic/yexception.h> +#include <util/generic/ptr.h> +#include <util/generic/utility.h> +#include <util/system/file.h> +#include <util/system/file_lock.h> +#include <util/system/fs.h> +#include <util/system/maxlen.h> +#include <util/system/mutex.h> +#include <util/system/utime.h> +#include <util/system/thread.h> + +#include <ydb/library/yql/core/spilling/interface/spilling.h> +#include <ydb/library/yql/core/spilling/storage/storage.h> + + +#include <ydb/library/yql/utils/log/log.h> + + +namespace NYql { +namespace NSpilling { + +// Class to implement ISpillStorage interface based on file system storage +class FileSpillStorage: public ISpillStorage { +public: + + ui64 GetCurrSize() {return 0;} // Returns current size of spill storage + // Returns full list of namespaces for current spill storage. Number of namespaces should be reasonable (no more than 1000) + TVector<TString> GetNamespaces(); + + // Returns list of file names for namespace + TVector<TString> GetNamespaceFiles(const TString& ns); + + THolder<ISpillFile> CreateSpillFile(const TString& ns, const TString& fn, ui32 reserveStep); + + TOperationResults LastOperationResults(); + FileSpillStorage (const TFileStorageConfig& config); + +private: + TFsPath RootPath_; // Root path to storage directory + TOperationResults OperationResults_; // Last operation results + + bool RootPathExists(); // Returns true if root path exists for spill storage + +}; + + +// Class to implement ISpillFile interface based on file system files. +class FsSpillFile: public ISpillFile { +public: + TString GetName() {return Name_;}; + bool IsLocked(); + ui64 Reserve(ui32 size); + void Write(ui32 offset, const char * data, ui32 bytes); + void Seek(ui32 offset); + i32 Read(ui32 offset, char* buf, ui32 len); + void Delete(); + FsSpillFile(const TString& name, EOpenMode oMode, ui32 reserveStep); +private: + TString Name_; + TFile File_; + TFileLock FileLock_; + bool Locked_ = false; + std::atomic<ui64> TotalSpace_ = 0; + std::atomic<ui64> ReservedSpace_ = 0; + ui32 ReserveStep_ = 10000000; // File is incremented by 10 MB chunks +}; + + +TVector<TString> FileSpillStorage::GetNamespaces() { + TVector<TString> res; + std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); + + if (!RootPathExists()) { + OperationResults_.Status = EOperationStatus::CannotOpenStorageProxy; + std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now(); + OperationResults_.ErrorString = "Root directory for temp storage path: " + TString(RootPath_) + " does not exist"; + return res; + } + + + TVector<TFsPath> dirEntries; + RootPath_.List(dirEntries); + for (const auto & p: dirEntries) { + if (p.IsDirectory() ) { + res.push_back(p.GetName()); + } + } + + OperationResults_.Status = EOperationStatus::Success; + OperationResults_.ErrorString.clear(); + + return res; + +} + +TVector<TString> FileSpillStorage::GetNamespaceFiles(const TString& ns) { + TVector<TString> res; + + if (!RootPathExists()) { + OperationResults_.Status = EOperationStatus::CannotOpenStorageProxy; + OperationResults_.ErrorString = "Root directory for temp storage path: " + TString(RootPath_) + " does not exist"; + return res; + } + + TFsPath child = RootPath_.Child(ns); + if ( !child.Exists()) { + return res; + } + TVector<TFsPath> dirEntries; + child.List(dirEntries); + for (const auto & p: dirEntries) { + TVector<TString> splitted; + size_t tokenCount = Split(p.GetName(), ".", splitted); + if (tokenCount >= 3 && tokenCount <= 4 && splitted[0] == "ydbspl" ) { + ui64 ind = 0; + if (TryFromString(splitted[1], ind)) { + } + res.push_back(p.GetName()); + } + + } + + OperationResults_.Status = EOperationStatus::Success; + OperationResults_.ErrorString.clear(); + + return res; + +} + +THolder<ISpillFile> FileSpillStorage::CreateSpillFile(const TString& ns, const TString& fn, ui32 reserveStep) { + TFsPath nsdir = RootPath_.Child(ns); + if (!nsdir.Exists()) { + nsdir.MkDir(); + } + TFsPath filePath = nsdir.Child(fn); + return MakeHolder<FsSpillFile>(filePath.GetPath(), + EOpenModeFlag::OpenAlways | EOpenModeFlag::RdWr , reserveStep); +} + + +bool FileSpillStorage::RootPathExists() { + if (RootPath_.IsDirectory()) { + return true; + } else { + return false; + } +} + +TOperationResults FileSpillStorage::LastOperationResults() { + return OperationResults_; +} + +FileSpillStorage::FileSpillStorage (const TFileStorageConfig& config) { + + std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); + + RootPath_ = TFsPath(config.Path); + + if (!RootPathExists()) { + OperationResults_.Status = EOperationStatus::CannotOpenStorageProxy; + OperationResults_.ErrorString = "Temp storage path: " + config.Path + " does not exist"; + return; + } + + + TVector<TFsPath> dirEntries; + RootPath_.List(dirEntries); + for (const auto & p: dirEntries) { + TVector<TString> splitted; + size_t tokenCount = Split(p.GetName(), ".", splitted); + if (tokenCount == 3 && splitted[0] == "ydbspl" ) { + ui64 ind = 0; + if (TryFromString(splitted[1], ind)) { + } + } + } + + OperationResults_.Status = EOperationStatus::Success; + std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now(); + OperationResults_.ErrorString.clear(); + + + +} + + +std::pair< THolder<ISpillStorage>, TOperationResults > OpenFileStorageForSpilling(const TFileStorageConfig & config ) { + THolder<FileSpillStorage> sp = MakeHolder<FileSpillStorage>(config); + TOperationResults res = sp->LastOperationResults(); + return std::make_pair< THolder<ISpillStorage>, TOperationResults >( std::move(sp), std::move(res) ); +} + + +bool FsSpillFile::IsLocked() { + return Locked_; +} + +ui64 FsSpillFile::Reserve(ui32 size) { + ui64 offset = ReservedSpace_.fetch_add(size); + while ( (offset + size) >= TotalSpace_ ) { + File_.Resize(offset + size + ReserveStep_); + TotalSpace_ = File_.GetLength(); + } + + return offset; +} + +void FsSpillFile::Write(ui32 offset, const char * data, ui32 bytes) { + File_.Pwrite(data, bytes, offset); +} + +void FsSpillFile::Seek(ui32 offset) { + File_.Seek(offset, SeekDir::sSet); +} + +i32 FsSpillFile::Read(ui32 offset, char* buf, ui32 len) { + return File_.Pread(buf, len, offset); +} + +void FsSpillFile::Delete() { + TFsPath path(Name_); + YQL_LOG(INFO) << "Deleting: " << Name_ << Endl; + path.ForceDelete(); +} + +FsSpillFile::FsSpillFile(const TString& name, EOpenMode oMode, ui32 reserveStep) : + Name_(name), + File_(name, oMode), + FileLock_(name), + ReserveStep_(reserveStep) +{ + Locked_ = FileLock_.TryAcquire(); + if (Locked_) { + ui64 fsize = File_.GetLength(); + if (fsize >= std::numeric_limits<ui32>::max()) { + + } + TotalSpace_ = (ui32) fsize; + ReservedSpace_ = fsize; + } +} + + +} +}
\ No newline at end of file diff --git a/ydb/library/yql/core/spilling/storage/file_storage/ya.make b/ydb/library/yql/core/spilling/storage/file_storage/ya.make new file mode 100644 index 00000000000..bb62430431c --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/file_storage/ya.make @@ -0,0 +1,32 @@ +LIBRARY() + +SRCS( + file_storage.cpp +) + +PEERDIR( + ydb/library/yql/utils/log +) + +NO_COMPILER_WARNINGS() + +IF (NOT MKQL_DISABLE_CODEGEN) + PEERDIR( + ydb/library/yql/minikql/codegen + contrib/libs/llvm12/lib/IR + contrib/libs/llvm12/lib/ExecutionEngine/MCJIT + contrib/libs/llvm12/lib/Linker + contrib/libs/llvm12/lib/Target/X86 + contrib/libs/llvm12/lib/Target/X86/AsmParser + contrib/libs/llvm12/lib/Transforms/IPO + ) +ELSE() + CFLAGS( + -DMKQL_DISABLE_CODEGEN + ) +ENDIF() + +YQL_LAST_ABI_VERSION() + +END() + diff --git a/ydb/library/yql/core/spilling/storage/storage.cpp b/ydb/library/yql/core/spilling/storage/storage.cpp new file mode 100644 index 00000000000..874e7ac91f5 --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/storage.cpp @@ -0,0 +1,141 @@ +#include "storage.h" + +#include <format> +#include <contrib/libs/xxhash/xxhash.h> +#include <ydb/library/yql/utils/yql_panic.h> +#include <ydb/library/yql/utils/log/log.h> + +namespace NYql { +namespace NSpilling { + +inline ui32 CalcUi32StrLen(ui32 bytes) { + if (!bytes) return 0; + return bytes / sizeof(ui32) + 1; +} + +EOperationType TSpillMetaRecord::GetOpType() { + ui32 opType = (Meta1_ & Meta1OperationBitMask) >> 28; // Calculating record type + return (EOperationType) opType; + +} + + +ui32 TSpillMetaRecord::GetNameSize() { + return (Meta1_ & (Meta1OperationStrLen) ); +} + + +ui32 TSpillMetaRecord::Size() { + ui32 res; + + res = 7 * sizeof(ui32); // Meta size + total record size + res += (CalcUi32StrLen(Name_.size() ) ) * sizeof(ui32); + + EOperationType eOpType = GetOpType(); + + switch (eOpType) { + case EOperationType::TimeMark: + res += 2 * sizeof(ui32); + break; + case EOperationType::StreamBufAdd: + res += 2 * sizeof(ui32); + break; + } + + return res; +} + +ui32 TSpillMetaRecord::DataSize() { + return DataSize_; +} + +ui32 TSpillMetaRecord::Offset() { + return Offset_; +} + +ui32 TSpillMetaRecord::DataHash() { + return DataHash_; +} + +void TSpillMetaRecord::Pack (TBuffer &buf ) { + ui32 size = Size(); + buf.Resize(size); + + ui32 * bufPtr = (ui32*) buf.Data(); + *(bufPtr) = RecordHash_; + *(bufPtr+1) = Offset_; + *(bufPtr+2) = RecordNumber_; + *(bufPtr+3) = Meta1_; + *(bufPtr+4) = DataSize_; + *(bufPtr+5) = DataHash_; + ui32 strUi32Size = CalcUi32StrLen(Name_.size()); + std::copy_n(Name_.data(), Name_.size(), buf.Data() + 6*sizeof(ui32) ); + *(bufPtr + 6 + strUi32Size ) = size; + XXH32_hash_t hash = XXH32( buf.Data() + sizeof(ui32), size - sizeof(ui32), 0); + *(bufPtr) = hash; + RecordHash_ = hash; + +} + +void TSpillMetaRecord::Unpack (TBuffer &buf ){ + ui32 * bufPtr = (ui32*) buf.Data(); + RecordHash_ = *(bufPtr); + Offset_ = *(bufPtr+1); + RecordNumber_ = *(bufPtr+2); + Meta1_ = *(bufPtr+3); + DataSize_ = *(bufPtr+4); + DataHash_ = *(bufPtr+5); + ui32 nameSize = GetNameSize(); + Name_.clear(); + Name_.append( buf.Data() + 6*sizeof(ui32) , nameSize ); + XXH32_hash_t hash = XXH32( buf.Data() + sizeof(ui32), Size() - sizeof(ui32), 0); + if (hash != RecordHash_) { + YQL_LOG(ERROR) << "Invalid hash: " << hash << " Record hash: " << RecordHash_ << Endl; + } +} + +TString TSpillMetaRecord::AsString() { + std::string res; + res = "RecordHash: " + std::to_string(RecordHash_) + " |Offset: " + std::to_string(Offset_) + " |RecordNumber: " + std::to_string(RecordNumber_) + " |Meta1: " + std::format("{:#08x}", Meta1_) + + " |DataSize: " + std::to_string(DataSize_) + " |DataHash: " + std::to_string(DataHash_) + " |Name: " + Name_; + + return std::move(res); + +} + + +void TSpillMetaRecord::SetOpType(EOperationType opType) { + ui32 opTypeVal = ((ui32) opType) << 28; +// Cout << "opTypeVal: " << opTypeVal << Endl; + Meta1_ = (opTypeVal | (Meta1_ & (~Meta1OperationBitMask) )); +} + + +void TSpillMetaRecord::SetNameSize() { + ui32 size = Name_.size(); + Meta1_ = (size | (Meta1_ & (~Meta1OperationStrLen) )); + +} + + +void TSpillMetaRecord::ScanForValidSpillRecords(TBuffer& buf, ui32& lastValidOffset, std::vector<TSpillMetaRecord>& records ) { + +} + +TSpillMetaRecord::TSpillMetaRecord(EOperationType opType, TString& name, ui32 offset, ui32 recordNum, ui32 dataSize, ui32 dataHash ) : + Name_(name), + Offset_(offset), + RecordNumber_(recordNum), + DataSize_(dataSize), + DataHash_(dataHash) +{ + SetOpType(opType); + SetNameSize(); + Bytes_ = Size(); +} + +TSpillMetaRecord::TSpillMetaRecord() {}; + +} +} + diff --git a/ydb/library/yql/core/spilling/storage/storage.h b/ydb/library/yql/core/spilling/storage/storage.h new file mode 100644 index 00000000000..f9dd97967ec --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/storage.h @@ -0,0 +1,121 @@ +#pragma once + +#include <library/cpp/threading/future/async.h> +#include <ydb/library/yql/core/spilling/interface/spilling.h> +#include <util/system/datetime.h> + +namespace NYql { +namespace NSpilling { + +class ISpillStorage; // Unified interface to implement spilling based on any supported storage type + +// Factory method to create ISpillStorage based on file system for usage. Error reasons are returned in TOperatonResults +std::pair< THolder<ISpillStorage>, TOperationResults > OpenFileStorageForSpilling(const TFileStorageConfig& config); + +const ui32 MagicForFileRecord32 = 0xD0F5F39F; // Magic header number to restore records in case of broken file + +// Defines possible operation types +enum class EOperationType { + Add = 1, // Add new temporary object + Delete = 2, // Delete temporary object + TimeMark = 3, // Add timemark to estimate timing of operations in meta file + StreamAdd = 4, // Add new stream sequence + StreamDelete = 5, // Deletes stream sequence + StreamBufAdd = 6, // Adding new stream buffer + SessionDelete = 7, // Deletes session and all objects associated with session +}; + + +// Storage meta record is used to describe particular spill object operation +class TSpillMetaRecord { +public: + EOperationType GetOpType(); // Returns operation type of the record + ui32 GetNameSize(); // Returns size of the record name + ui32 Size(); // Total size of record in bytes + ui32 DataSize(); // Size of the record Data + ui32 Offset(); // Offset of the data in data file + ui32 DataHash(); // Returns data hash + void Pack (TBuffer& buf ); // Serializes record to buf. After the call Buf is ready to be written to spill meta file + void Unpack (TBuffer& buf ); // Restores internal state from buf. + TString AsString(); // Returns string representation of SpillMetaRecord + void SetDataHash(const ui32 hash) {DataHash_ = hash;} + void SetObjId(const ui32 objId) {RecordNumber_ = objId;} + // Scans buf for all valid spill meta records and return all recods found in record argument. lastValidOffset - last byte offset of last valid record in buf + static void ScanForValidSpillRecords(TBuffer& buf, ui32& lastValidOffset, std::vector<TSpillMetaRecord>& records ); + TSpillMetaRecord(EOperationType opType, TString& name, ui32 offset, ui32 recordNum, ui32 dataSize, ui32 dataHash ); + TSpillMetaRecord(); +private: + void SetOpType(EOperationType opType = EOperationType::Add); // Defines operation type of the record + void SetNameSize(); // Sets str size in bytes in Meta1 record + ui32 RecordHash_ = 0; // Hash number to check consistency of record + ui32 Offset_ = 0; // Offset of the object in the data file + ui32 RecordNumber_ = 0; // Global increasing record number for particular namespace + ui32 Meta1_ = 0; // Meta information about the record. See below description of bit masks to unpack info from this field + ui32 DataSize_ = 0; // Data size of the buffer in dat file associated with record + ui32 DataHash_ = 0; // Hash of buffer in dat file + std::string Name_; // Name of the object + ui32 Bytes_ = 0; // Total bytes of spill meta record in meta file (28 + size of Name rounded to 4) + +}; + +struct TSpillStreamBufRecord : public TSpillMetaRecord { + ui32 StreamId = 0; // Record id of the stream + ui32 BufId = 0; // Id of the buffer inside the stream +}; + +const ui32 Meta1OperationBitMask = 0xF0000000; // Defines operation type stored in the record +const ui32 Meta1OperationStrLen = 0x0000FFFF; // Bit mask to extract length of spill object name (maximum 4095 bytes) +const ui32 Meta1OperationRetention = 0x0FF00000; // Retention period in hours. Value 0xFF (255) means unlimited retention period, it should be deleted manually + + +// Time mark record to estimate in between records addition time +struct TTimeMarkRecord : public TSpillMetaRecord { + ui32 Microseconds1 = 0; // Current Microseconds time since epoch (minor 32 bits) + ui32 Microseconds2 = 0; // Current Microseconds time since epoch (major 32 bits) +}; + +// Interface for spill file for particular namespace +class ISpillFile { +public: + virtual TString GetName() = 0; // Returns full name of the file + virtual bool IsLocked() = 0; // True when file is locked + virtual ui64 Reserve(ui32 size) = 0; // Reserves size bytes for writing, returns file offset + virtual void Write(ui32 offset, const char * data, ui32 bytes) = 0; + virtual void Seek(ui32 offset) = 0; + // Reads up to 1 GB without retrying, returns -1 on error + virtual i32 Read(ui32 offset, char* buf, ui32 len) = 0; + virtual void Delete() = 0; // Deletes spill file + virtual ~ISpillFile() = default; +}; + + +// Interface to work with particular +class ISpillStorage { +public: + + virtual ui64 GetCurrSize() = 0; // Returns current size of spill storage + + // Returns full list of namespaces for current spill storage. + // All namespaces are on the same level and don't form hierarchy. + // Number of namespaces should be reasonable (no more than 1000) + // Namespace name should be alphanumerical only. + virtual TVector<TString> GetNamespaces() = 0; + + // Returns list of file names for namespace + virtual TVector<TString> GetNamespaceFiles(const TString& ns) = 0; + + // Creates file in namespace ns with name fn. File interface ISpillFile is ready for writing and locked in case of success. + // Caller should check success with LastOperationResults. reserveStep is a step to increase allocated file storage. + virtual THolder<ISpillFile> CreateSpillFile(const TString& ns, const TString& fn, ui32 reserveStep) = 0; + + // Returns last operation results + virtual TOperationResults LastOperationResults() = 0; + + + virtual ~ISpillStorage() = default; + +}; + + +} +}
\ No newline at end of file diff --git a/ydb/library/yql/core/spilling/storage/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..649467a4f98 --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,78 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-core-spilling-storage-ut) +target_compile_options(ydb-library-yql-core-spilling-storage-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-core-spilling-storage-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage +) +target_link_libraries(ydb-library-yql-core-spilling-storage-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + core-spilling-storage + yql-utils-log +) +target_link_options(ydb-library-yql-core-spilling-storage-ut PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-library-yql-core-spilling-storage-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/ut/storage_ut.cpp +) +set_property( + TARGET + ydb-library-yql-core-spilling-storage-ut + PROPERTY + SPLIT_FACTOR + 60 +) +add_yunittest( + NAME + ydb-library-yql-core-spilling-storage-ut + TEST_TARGET + ydb-library-yql-core-spilling-storage-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-storage-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-storage-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-storage-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-core-spilling-storage-ut + system_allocator +) +vcs_info(ydb-library-yql-core-spilling-storage-ut) diff --git a/ydb/library/yql/core/spilling/storage/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..44f49c0573d --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.linux-aarch64.txt @@ -0,0 +1,81 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-core-spilling-storage-ut) +target_compile_options(ydb-library-yql-core-spilling-storage-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-core-spilling-storage-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage +) +target_link_libraries(ydb-library-yql-core-spilling-storage-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + core-spilling-storage + yql-utils-log +) +target_link_options(ydb-library-yql-core-spilling-storage-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-core-spilling-storage-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/ut/storage_ut.cpp +) +set_property( + TARGET + ydb-library-yql-core-spilling-storage-ut + PROPERTY + SPLIT_FACTOR + 60 +) +add_yunittest( + NAME + ydb-library-yql-core-spilling-storage-ut + TEST_TARGET + ydb-library-yql-core-spilling-storage-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-storage-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-storage-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-storage-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-core-spilling-storage-ut + cpp-malloc-jemalloc +) +vcs_info(ydb-library-yql-core-spilling-storage-ut) diff --git a/ydb/library/yql/core/spilling/storage/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..3d9aeb7cdef --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.linux-x86_64.txt @@ -0,0 +1,83 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-core-spilling-storage-ut) +target_compile_options(ydb-library-yql-core-spilling-storage-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-core-spilling-storage-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage +) +target_link_libraries(ydb-library-yql-core-spilling-storage-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + core-spilling-storage + yql-utils-log +) +target_link_options(ydb-library-yql-core-spilling-storage-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-core-spilling-storage-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/ut/storage_ut.cpp +) +set_property( + TARGET + ydb-library-yql-core-spilling-storage-ut + PROPERTY + SPLIT_FACTOR + 60 +) +add_yunittest( + NAME + ydb-library-yql-core-spilling-storage-ut + TEST_TARGET + ydb-library-yql-core-spilling-storage-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-storage-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-storage-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-storage-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-core-spilling-storage-ut + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(ydb-library-yql-core-spilling-storage-ut) diff --git a/ydb/library/yql/core/spilling/storage/ut/CMakeLists.txt b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/core/spilling/storage/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..0185830a78a --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.windows-x86_64.txt @@ -0,0 +1,71 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-core-spilling-storage-ut) +target_compile_options(ydb-library-yql-core-spilling-storage-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-core-spilling-storage-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage +) +target_link_libraries(ydb-library-yql-core-spilling-storage-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + core-spilling-storage + yql-utils-log +) +target_sources(ydb-library-yql-core-spilling-storage-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/ut/storage_ut.cpp +) +set_property( + TARGET + ydb-library-yql-core-spilling-storage-ut + PROPERTY + SPLIT_FACTOR + 60 +) +add_yunittest( + NAME + ydb-library-yql-core-spilling-storage-ut + TEST_TARGET + ydb-library-yql-core-spilling-storage-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-storage-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-storage-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-storage-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-core-spilling-storage-ut + system_allocator +) +vcs_info(ydb-library-yql-core-spilling-storage-ut) diff --git a/ydb/library/yql/core/spilling/storage/ut/storage_ut.cpp b/ydb/library/yql/core/spilling/storage/ut/storage_ut.cpp new file mode 100644 index 00000000000..1c42e106292 --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/ut/storage_ut.cpp @@ -0,0 +1,104 @@ +#include <ydb/library/yql/minikql/mkql_runtime_version.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <chrono> +#include <iostream> +#include <cstring> +#include <vector> +#include <cassert> + +#include <util/system/compiler.h> +#include <util/stream/null.h> +#include <util/system/fs.h> + +#include <cstdint> + +#include <ydb/library/yql/core/spilling/interface/spilling.h> +#include <ydb/library/yql/core/spilling/storage/storage.h> + +namespace NYql { +namespace NSpilling { + +using namespace NSpilling; + +constexpr bool IsVerbose = true; +#define CTEST (IsVerbose ? Cerr : Cnull) + + +Y_UNIT_TEST_SUITE(TSpillingTest) { + +Y_UNIT_TEST(TestCreateStorage) { + TFileStorageConfig config; + config.Path = NFs::CurrentWorkingDirectory(); + std::pair< THolder<ISpillStorage>, TOperationResults > sp = OpenFileStorageForSpilling(config); + CTEST << "Path: " << config.Path << Endl; + UNIT_ASSERT(sp.second.Status == EOperationStatus::Success); +} + + +Y_UNIT_TEST(TestCreateNoStorage) { + TFileStorageConfig config; + config.Path = "Temp123456"; + std::pair< THolder<ISpillStorage>, TOperationResults > sp = OpenFileStorageForSpilling(config); + CTEST << "Path: " << config.Path << Endl; + CTEST << "Error string: " << sp.second.ErrorString << Endl; + UNIT_ASSERT(sp.second.Status == EOperationStatus::CannotOpenStorageProxy); +} + +Y_UNIT_TEST(TestNamespaces) { + TFileStorageConfig config; + config.Path = NFs::CurrentWorkingDirectory(); + std::pair< THolder<ISpillStorage>, TOperationResults > sp = OpenFileStorageForSpilling(config); + TVector<TString> ns = sp.first->GetNamespaces(); + for ( auto n : ns) { + CTEST << "Namespace: " << n << Endl; + TVector<TString> nsf = sp.first->GetNamespaceFiles(n); + TOperationResults opres = sp.first->LastOperationResults(); + for ( auto n : nsf) { + CTEST << "File: " << n << Endl; + } + UNIT_ASSERT(opres.Status == EOperationStatus::Success); + } + UNIT_ASSERT(sp.second.Status == EOperationStatus::Success); + UNIT_ASSERT(ns.size() != 0); +} + + +Y_UNIT_TEST(TestRecordSizes) { + size_t s1 = sizeof(TSpillMetaRecord); + size_t s2 = sizeof(TTimeMarkRecord); + CTEST << "SpillMetaRecord bytes: " << s1 << Endl; + CTEST << "TimeMarkRecord bytes: " << s2 << Endl; + UNIT_ASSERT(s1+8 == s2); +} + +Y_UNIT_TEST(TestCreateNamespaces) { + + TFileStorageConfig config; + config.Path = NFs::CurrentWorkingDirectory(); + std::pair< THolder<ISpillStorage>, TOperationResults > sp = OpenFileStorageForSpilling(config); + TVector<TString> ns = sp.first->GetNamespaces(); + for ( auto n : ns) { + CTEST << "Namespace: " << n << Endl; + TVector<TString> nsf = sp.first->GetNamespaceFiles(n); + TOperationResults opres = sp.first->LastOperationResults(); + for ( auto n : nsf) { + CTEST << "File: " << n << Endl; + } + UNIT_ASSERT(opres.Status == EOperationStatus::Success); + } + THolder<ISpillFile> spf = sp.first->CreateSpillFile(TString("temp1"), TString("ydbspl.1.met"), 4*1024*1024); + TOperationResults opres = sp.first->LastOperationResults(); + UNIT_ASSERT(opres.Status == EOperationStatus::Success); + UNIT_ASSERT(spf->IsLocked()); + +} + + + +} + +} +} + + diff --git a/ydb/library/yql/core/spilling/storage/ut/ya.make b/ydb/library/yql/core/spilling/storage/ut/ya.make new file mode 100644 index 00000000000..51f7c47eeb3 --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/ut/ya.make @@ -0,0 +1,32 @@ +UNITTEST_FOR(ydb/library/yql/core/spilling/storage) + +FORK_SUBTESTS() + +SPLIT_FACTOR(60) + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + TIMEOUT(3600) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +SRCS( + storage_ut.cpp + ) + +PEERDIR( + ydb/library/yql/utils/log +) + +YQL_LAST_ABI_VERSION() + +IF (MKQL_RUNTIME_VERSION) + CFLAGS( + -DMKQL_RUNTIME_VERSION=$MKQL_RUNTIME_VERSION + ) +ENDIF() + +END() diff --git a/ydb/library/yql/core/spilling/storage/ya.make b/ydb/library/yql/core/spilling/storage/ya.make new file mode 100644 index 00000000000..8c4746a1a39 --- /dev/null +++ b/ydb/library/yql/core/spilling/storage/ya.make @@ -0,0 +1,36 @@ +LIBRARY() + +SRCS( + storage.h + storage.cpp +) + +PEERDIR( + ydb/library/yql/core/spilling/storage/file_storage +) + +NO_COMPILER_WARNINGS() + +IF (NOT MKQL_DISABLE_CODEGEN) + PEERDIR( + ydb/library/yql/minikql/codegen + contrib/libs/llvm12/lib/IR + contrib/libs/llvm12/lib/ExecutionEngine/MCJIT + contrib/libs/llvm12/lib/Linker + contrib/libs/llvm12/lib/Target/X86 + contrib/libs/llvm12/lib/Target/X86/AsmParser + contrib/libs/llvm12/lib/Transforms/IPO + ) +ELSE() + CFLAGS( + -DMKQL_DISABLE_CODEGEN + ) +ENDIF() + +YQL_LAST_ABI_VERSION() + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/library/yql/core/spilling/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/spilling/ut/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..ba4e853e58f --- /dev/null +++ b/ydb/library/yql/core/spilling/ut/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,79 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-core-spilling-ut) +target_compile_options(ydb-library-yql-core-spilling-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-core-spilling-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling +) +target_link_libraries(ydb-library-yql-core-spilling-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-core-spilling + yql-public-udf + udf-service-exception_policy +) +target_link_options(ydb-library-yql-core-spilling-ut PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-library-yql-core-spilling-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/ut/spilling_ut.cpp +) +set_property( + TARGET + ydb-library-yql-core-spilling-ut + PROPERTY + SPLIT_FACTOR + 60 +) +add_yunittest( + NAME + ydb-library-yql-core-spilling-ut + TEST_TARGET + ydb-library-yql-core-spilling-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-core-spilling-ut + system_allocator +) +vcs_info(ydb-library-yql-core-spilling-ut) diff --git a/ydb/library/yql/core/spilling/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/spilling/ut/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..0fe44c24967 --- /dev/null +++ b/ydb/library/yql/core/spilling/ut/CMakeLists.linux-aarch64.txt @@ -0,0 +1,82 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-core-spilling-ut) +target_compile_options(ydb-library-yql-core-spilling-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-core-spilling-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling +) +target_link_libraries(ydb-library-yql-core-spilling-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + yql-core-spilling + yql-public-udf + udf-service-exception_policy +) +target_link_options(ydb-library-yql-core-spilling-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-core-spilling-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/ut/spilling_ut.cpp +) +set_property( + TARGET + ydb-library-yql-core-spilling-ut + PROPERTY + SPLIT_FACTOR + 60 +) +add_yunittest( + NAME + ydb-library-yql-core-spilling-ut + TEST_TARGET + ydb-library-yql-core-spilling-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-core-spilling-ut + cpp-malloc-jemalloc +) +vcs_info(ydb-library-yql-core-spilling-ut) diff --git a/ydb/library/yql/core/spilling/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/spilling/ut/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..33abfe68c08 --- /dev/null +++ b/ydb/library/yql/core/spilling/ut/CMakeLists.linux-x86_64.txt @@ -0,0 +1,84 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-core-spilling-ut) +target_compile_options(ydb-library-yql-core-spilling-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-core-spilling-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling +) +target_link_libraries(ydb-library-yql-core-spilling-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-core-spilling + yql-public-udf + udf-service-exception_policy +) +target_link_options(ydb-library-yql-core-spilling-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-core-spilling-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/ut/spilling_ut.cpp +) +set_property( + TARGET + ydb-library-yql-core-spilling-ut + PROPERTY + SPLIT_FACTOR + 60 +) +add_yunittest( + NAME + ydb-library-yql-core-spilling-ut + TEST_TARGET + ydb-library-yql-core-spilling-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-core-spilling-ut + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(ydb-library-yql-core-spilling-ut) diff --git a/ydb/library/yql/core/spilling/ut/CMakeLists.txt b/ydb/library/yql/core/spilling/ut/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/core/spilling/ut/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/core/spilling/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/spilling/ut/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..6d5ce50c09b --- /dev/null +++ b/ydb/library/yql/core/spilling/ut/CMakeLists.windows-x86_64.txt @@ -0,0 +1,72 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-core-spilling-ut) +target_compile_options(ydb-library-yql-core-spilling-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-core-spilling-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling +) +target_link_libraries(ydb-library-yql-core-spilling-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-core-spilling + yql-public-udf + udf-service-exception_policy +) +target_sources(ydb-library-yql-core-spilling-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/ut/spilling_ut.cpp +) +set_property( + TARGET + ydb-library-yql-core-spilling-ut + PROPERTY + SPLIT_FACTOR + 60 +) +add_yunittest( + NAME + ydb-library-yql-core-spilling-ut + TEST_TARGET + ydb-library-yql-core-spilling-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-core-spilling-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-core-spilling-ut + system_allocator +) +vcs_info(ydb-library-yql-core-spilling-ut) diff --git a/ydb/library/yql/core/spilling/ut/spilling_ut.cpp b/ydb/library/yql/core/spilling/ut/spilling_ut.cpp new file mode 100644 index 00000000000..d189a503a14 --- /dev/null +++ b/ydb/library/yql/core/spilling/ut/spilling_ut.cpp @@ -0,0 +1,368 @@ +#include <ydb/library/yql/minikql/mkql_runtime_version.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <chrono> +#include <iostream> +#include <cstring> +#include <vector> +#include <cassert> +#include <thread> + +#include <util/system/fs.h> +#include <util/system/compiler.h> +#include <util/stream/null.h> +#include <util/system/mem_info.h> + +#include <cstdint> + +#include <ydb/library/yql/core/spilling/interface/spilling.h> +#include <ydb/library/yql/core/spilling/storage/storage.h> + +namespace NYql { +namespace NSpilling { + +using namespace NSpilling; +using namespace std::chrono_literals; + +constexpr bool IsVerbose = true; +#define CTEST (IsVerbose ? Cerr : Cnull) + + +Y_UNIT_TEST_SUITE(TYDBLibrarySpillingTest) { + + Y_UNIT_TEST(TestCreateProxy) { + TFileStorageConfig config; + config.Path = NFs::CurrentWorkingDirectory(); + TTempStorageExecutionPolicy policy; + std::pair < THolder<ITempStorageProxy>, TOperationResults> tproxy = CreateFileStorageProxy(config, policy ); + CTEST << "Status: " << ui32( tproxy.second.Status ) << Endl; + UNIT_ASSERT(tproxy.second.Status == EOperationStatus::Success); + } + + Y_UNIT_TEST(TestCreateSession) { + TFileStorageConfig config; + config.Path = NFs::CurrentWorkingDirectory(); + TTempStorageExecutionPolicy policy; + std::pair < THolder<ITempStorageProxy>, TOperationResults> tproxy = CreateFileStorageProxy(config, policy ); + THolder<ISession> session = tproxy.first->CreateSession(); + UNIT_ASSERT(session != nullptr); + } + + Y_UNIT_TEST(TestSave) { + TFileStorageConfig config; + config.Path = NFs::CurrentWorkingDirectory(); + TTempStorageExecutionPolicy policy; + std::pair < THolder<ITempStorageProxy>, TOperationResults> tproxy = CreateFileStorageProxy(config, policy ); + THolder<ISession> session = tproxy.first->CreateSession(); + NThreading::TFuture<TOperationResults> ftr; + const ui32 bufSize = 1024*1024; + const ui32 iters = 1000; + std::vector<TBuffer> buffers, buffers1; + + NMemInfo::TMemInfo mi = NMemInfo::GetMemInfo(); + CTEST << "Mem usage before buffs prepare (MB): " << mi.RSS / (1024 * 1024) << Endl; + std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); + for (ui32 i = 0; i < iters; i++) { + TBuffer buf(bufSize); + buf.Resize(bufSize); + memset(buf.Data(), i+1, bufSize/sizeof(int) - 1 ); + buffers1.emplace_back(std::move(buf)); + } + mi = NMemInfo::GetMemInfo(); + CTEST << "Mem usage after buffs prepare (MB): " << mi.RSS / (1024 * 1024) << Endl; + std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now(); + ui64 execTime = std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count(); + CTEST << "Execution time for " << iters << " prepare buf calls (microseconds): " << execTime << Endl; + + begin = std::chrono::steady_clock::now(); + for (ui32 i = 0; i < iters; i++) { + buffers.emplace_back(std::move(buffers1[i])); + } + end = std::chrono::steady_clock::now(); + execTime = std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count(); + CTEST << "Execution time for " << iters << " moving buffers: (microseconds): " << execTime << Endl; + + + begin = std::chrono::steady_clock::now(); + for (ui32 i = 0; i < iters; i++) { + ftr = session->Save(TString("Test"), TString("test" + std::to_string(i)), std::move(buffers[i])); + } + end = std::chrono::steady_clock::now(); + execTime = std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count(); + CTEST << "Execution time for " << iters << " save calls (microseconds): " << execTime << Endl; + CTEST << "Per one call (microseconds): " << execTime/iters << Endl; + + mi = NMemInfo::GetMemInfo(); + CTEST << "Mem usage after buffs save (MB): " << mi.RSS / (1024 * 1024) << Endl; + + TSessionDataStat dstat = session->GetSessionDataStat(); + CTEST << "Session total data: " << dstat.Provided << Endl; + CTEST << "Session spilled data: " << dstat.Spilled << Endl; + CTEST << "Session in memory data: " << dstat.InMemory << Endl; + + std::this_thread::sleep_for(5000ms); + + dstat = session->GetSessionDataStat(); + CTEST << "Session spilled data after sleep: " << dstat.Spilled << Endl; + CTEST << "Session loaded from memory data: " << dstat.LoadedFromMemory << Endl; + CTEST << "Session in memory data: " << dstat.InMemory << Endl; + + mi = NMemInfo::GetMemInfo(); + CTEST << "Mem usage after buffs writing (MB): " << mi.RSS / (1024 * 1024) << Endl; + + UNIT_ASSERT(ftr.Initialized()); + } + + Y_UNIT_TEST(TestLoad) { + TFileStorageConfig config; + config.Path = NFs::CurrentWorkingDirectory(); + TTempStorageExecutionPolicy policy; + std::pair < THolder<ITempStorageProxy>, TOperationResults> tproxy = CreateFileStorageProxy(config, policy ); + THolder<ISession> session = tproxy.first->CreateSession(); + + NThreading::TFuture<TOperationResults> ftr; + std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); + const ui32 bufSize = 1024*1024; + const ui32 iters = 1000; + for (ui32 i = 0; i < iters; i++) { + TBuffer buf(bufSize); + buf.Resize(bufSize); + memset(buf.Data(), i+1, bufSize/sizeof(int) - 1 ); + ftr = session->Save(TString("Test"), TString("test" + std::to_string(i)), std::move(buf)); + } + std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now(); + ui64 execTime = std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count(); + CTEST << "Execution time for " << iters << " save calls (microseconds): " << execTime << Endl; + CTEST << "Per one call (microseconds): " << execTime/iters << Endl; + +// std::this_thread::sleep_for(3000ms); + + begin = std::chrono::steady_clock::now(); + NThreading::TFuture<TLoadOperationResults> ftrl; + for (ui32 i = 0; i < iters; i++) { + ftrl = session->Load(TString("Test"), TString("test" + std::to_string(i))); + } + end = std::chrono::steady_clock::now(); + execTime = std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count(); + CTEST << "Execution time for " << iters << " load calls (microseconds): " << execTime << Endl; + CTEST << "Per one call (microseconds): " << execTime/iters << Endl; + + TSessionDataStat dstat = session->GetSessionDataStat(); + CTEST << "Session total data: " << dstat.Provided << Endl; + CTEST << "Session spilled data: " << dstat.Spilled << Endl; + CTEST << "Session loaded from memory data: " << dstat.LoadedFromMemory << Endl; + CTEST << "Session in memory data: " << dstat.InMemory << Endl; + + UNIT_ASSERT(ftrl.Initialized()); + } + + Y_UNIT_TEST(TestSetOp) { + TString name{"Test"}; + TSpillMetaRecord mr{EOperationType::Add, name, 0, 0, 1, 2}; + auto res = mr.GetOpType(); + CTEST << "OpType: " << (ui32) res << Endl; + CTEST << mr.AsString() << Endl; + UNIT_ASSERT(res == EOperationType::Add); + + } + + Y_UNIT_TEST(TestSpillMetaPack) { + TString name{"Test1"}; + TSpillMetaRecord mr{EOperationType::Add, name, 1, 2, 3, 4}; + auto res = mr.GetOpType(); + CTEST << mr.AsString() << Endl; + TBuffer buf; + mr.Pack(buf); + CTEST << "Buf size: " << buf.Size() << Endl; + + TString name1{"Test22222"}; + TSpillMetaRecord mr1{EOperationType::Add, name1, 2, 3, 5, 6}; + CTEST << mr1.AsString() << Endl; + + mr1.Unpack(buf); + CTEST << mr1.AsString() << Endl; + + UNIT_ASSERT(res == EOperationType::Add); + + } + + Y_UNIT_TEST(TestSpillSaveLoad) { + TFileStorageConfig config; + config.Path = NFs::CurrentWorkingDirectory(); + TTempStorageExecutionPolicy policy; + std::pair < THolder<ITempStorageProxy>, TOperationResults> tproxy = CreateFileStorageProxy(config, policy ); + THolder<ISession> session = tproxy.first->CreateSession(); + + NThreading::TFuture<TOperationResults> ftr; + std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); + const ui32 bufSize = 1024*1024; + const ui32 iters = 1000; + for (ui32 i = 0; i < iters; i++) { + TBuffer buf(bufSize); + buf.Resize(bufSize); + memset(buf.Data(), i+1, bufSize/sizeof(int) - 1 ); + ftr = session->Save(TString("Test"), TString("test" + std::to_string(i)), std::move(buf)); + } + std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now(); + ui64 execTime = std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count(); + CTEST << "Execution time for " << iters << " save calls (microseconds): " << execTime << Endl; + CTEST << "Per one call (microseconds): " << execTime/iters << Endl; + + std::this_thread::sleep_for(3000ms); + + begin = std::chrono::steady_clock::now(); + NThreading::TFuture<TLoadOperationResults> ftrl; + for (ui32 i = 0; i < iters; i++) { + ftrl = session->Load(TString("Test"), TString("test" + std::to_string(i))); + } + end = std::chrono::steady_clock::now(); + execTime = std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count(); + CTEST << "Execution time for " << iters << " load calls (microseconds): " << execTime << Endl; + CTEST << "Per one call (microseconds): " << execTime/iters << Endl; + + std::this_thread::sleep_for(6000ms); + + TSessionDataStat dstat = session->GetSessionDataStat(); + CTEST << "Session total data: " << dstat.Provided << Endl; + CTEST << "Session spilled data: " << dstat.Spilled << Endl; + CTEST << "Session loaded from memory data: " << dstat.LoadedFromMemory << Endl; + CTEST << "Session loaded from storage data: " << dstat.LoadedFromStorage << Endl; + CTEST << "Session in memory data: " << dstat.InMemory << Endl; + + UNIT_ASSERT(ftrl.Initialized()); + + } + + Y_UNIT_TEST(TestGarbageCollection) { + TFileStorageConfig config; + config.Path = NFs::CurrentWorkingDirectory(); + TTempStorageExecutionPolicy policy; + std::pair < THolder<ITempStorageProxy>, TOperationResults> tproxy = CreateFileStorageProxy(config, policy ); + for (ui32 sessNum = 0; sessNum < 10; sessNum++) { + THolder<ISession> session = tproxy.first->CreateSession(); + + NThreading::TFuture<TOperationResults> ftr; + std::chrono::steady_clock::time_point begin = + std::chrono::steady_clock::now(); + const ui32 bufSize = 1024 * 1024; + const ui32 iters = 1000; + for (ui32 i = 0; i < iters; i++) { + TBuffer buf(bufSize); + buf.Resize(bufSize); + memset(buf.Data(), i + 1, bufSize / sizeof(int) - 1); + ftr = session->Save(TString("Test"), + TString("test" + std::to_string(i)), + std::move(buf)); + } + std::chrono::steady_clock::time_point end = + std::chrono::steady_clock::now(); + ui64 execTime = + std::chrono::duration_cast<std::chrono::microseconds>(end - + begin) + .count(); + CTEST << "Execution time for " << iters + << " save calls (microseconds): " << execTime << Endl; + CTEST << "Per one call (microseconds): " << execTime / iters + << Endl; + + std::this_thread::sleep_for(3000ms); + + TSessionDataStat dstat = session->GetSessionDataStat(); + CTEST << "Session total data after save: " << dstat.Provided << Endl; + CTEST << "Session spilled data after save: " << dstat.Spilled << Endl; + CTEST << "Session loaded from memory data after save : " + << dstat.LoadedFromMemory << Endl; + CTEST << "Session loaded from storage data after save: " + << dstat.LoadedFromStorage << Endl; + CTEST << "Session in memory data after save: " << dstat.InMemory << Endl; + + + begin = std::chrono::steady_clock::now(); + NThreading::TFuture<TLoadOperationResults> ftrl; + for (ui32 i = 0; i < iters; i++) { + ftrl = session->Load(TString("Test"), + TString("test" + std::to_string(i))); + } + end = std::chrono::steady_clock::now(); + execTime = std::chrono::duration_cast<std::chrono::microseconds>( + end - begin) + .count(); + CTEST << "Execution time for " << iters + << " load calls (microseconds): " << execTime << Endl; + CTEST << "Per one call (microseconds): " << execTime / iters + << Endl; + + std::this_thread::sleep_for(6000ms); + + dstat = session->GetSessionDataStat(); + CTEST << "Session total data: " << dstat.Provided << Endl; + CTEST << "Session spilled data: " << dstat.Spilled << Endl; + CTEST << "Session loaded from memory data: " + << dstat.LoadedFromMemory << Endl; + CTEST << "Session loaded from storage data: " + << dstat.LoadedFromStorage << Endl; + CTEST << "Session in memory data: " << dstat.InMemory << Endl; + + UNIT_ASSERT(ftrl.Initialized()); + } + } + + + Y_UNIT_TEST(TestSpillStreamSaveLoad) { + TFileStorageConfig config; + config.Path = NFs::CurrentWorkingDirectory(); + TTempStorageExecutionPolicy policy; + std::pair < THolder<ITempStorageProxy>, TOperationResults> tproxy = CreateFileStorageProxy(config, policy ); + THolder<ISession> session = tproxy.first->CreateSession(); + + NThreading::TFuture<TOperationResults> ftr; + std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); + const ui32 bufSize = 1024*1024; + const ui32 iters = 1000; + auto res = session->OpenStream(TString("Test"), TString("Stream1")); + auto st1 = std::move(res.first); + CTEST << "Stream size before save: " << st1->Size() << Endl; + for (ui32 i = 0; i < iters; i++) { + TBuffer buf(bufSize); + buf.Resize(bufSize); + memset(buf.Data(), i+1, bufSize/sizeof(int) - 1 ); + ftr = st1->Save(std::move(buf)); + } + CTEST << "Stream size after save: " << st1->Size() << Endl; + std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now(); + ui64 execTime = std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count(); + CTEST << "Execution time for " << iters << " save calls (microseconds): " << execTime << Endl; + CTEST << "Per one call (microseconds): " << execTime/iters << Endl; + + std::this_thread::sleep_for(3000ms); + + begin = std::chrono::steady_clock::now(); + NThreading::TFuture<TLoadOperationResults> ftrl; + for (ui32 i = 0; i < iters; i++) { + ftrl = st1->Load(i); + } + end = std::chrono::steady_clock::now(); + execTime = std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count(); + CTEST << "Execution time for " << iters << " load calls (microseconds): " << execTime << Endl; + CTEST << "Per one call (microseconds): " << execTime/iters << Endl; + + std::this_thread::sleep_for(6000ms); + + TSessionDataStat dstat = session->GetSessionDataStat(); + CTEST << "Session total data: " << dstat.Provided << Endl; + CTEST << "Session spilled data: " << dstat.Spilled << Endl; + CTEST << "Session loaded from memory data: " << dstat.LoadedFromMemory << Endl; + CTEST << "Session loaded from storage data: " << dstat.LoadedFromStorage << Endl; + CTEST << "Session in memory data: " << dstat.InMemory << Endl; + + UNIT_ASSERT(ftrl.Initialized()); + + } + + + +} + +} + +} diff --git a/ydb/library/yql/core/spilling/ut/ya.make b/ydb/library/yql/core/spilling/ut/ya.make new file mode 100644 index 00000000000..7c97cbed1b8 --- /dev/null +++ b/ydb/library/yql/core/spilling/ut/ya.make @@ -0,0 +1,35 @@ +UNITTEST_FOR(ydb/library/yql/core/spilling) + +FORK_SUBTESTS() + +SPLIT_FACTOR(60) + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + TIMEOUT(3600) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +SRCS( + spilling_ut.cpp + ) + +PEERDIR( + ydb/library/yql/public/udf + ydb/library/yql/public/udf/service/exception_policy +) + +YQL_LAST_ABI_VERSION() + +IF (MKQL_RUNTIME_VERSION) + CFLAGS( + -DMKQL_RUNTIME_VERSION=$MKQL_RUNTIME_VERSION + ) +ENDIF() + +REQUIREMENTS(ram:10) + +END() diff --git a/ydb/library/yql/core/spilling/ya.make b/ydb/library/yql/core/spilling/ya.make new file mode 100644 index 00000000000..af3aa1592c6 --- /dev/null +++ b/ydb/library/yql/core/spilling/ya.make @@ -0,0 +1,52 @@ +LIBRARY() + +SRCS( + spilling_imp.cpp + spilling_imp.h + namespaces_list.h + namespaces_list.cpp + namespace_cache.h + namespace_cache.cpp + interface/spilling.h + storage/file_storage/file_storage.cpp + storage/storage.h + storage/storage.cpp +) + +PEERDIR( + contrib/libs/apache/arrow + ydb/library/binary_json + ydb/library/yql/utils + ydb/library/yql/utils/log + ydb/library/yql/core/spilling/storage +) + +NO_COMPILER_WARNINGS() + +IF (NOT MKQL_DISABLE_CODEGEN) + PEERDIR( + ydb/library/yql/minikql/codegen + contrib/libs/llvm12/lib/IR + contrib/libs/llvm12/lib/ExecutionEngine/MCJIT + contrib/libs/llvm12/lib/Linker + contrib/libs/llvm12/lib/Target/X86 + contrib/libs/llvm12/lib/Target/X86/AsmParser + contrib/libs/llvm12/lib/Transforms/IPO + ) +ELSE() + CFLAGS( + -DMKQL_DISABLE_CODEGEN + ) +ENDIF() + +YQL_LAST_ABI_VERSION() + +END() + +RECURSE( + storage +) + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/library/yql/core/url_lister/ya.make b/ydb/library/yql/core/url_lister/ya.make index 2caf6cf6f0c..b11cf9dda46 100644 --- a/ydb/library/yql/core/url_lister/ya.make +++ b/ydb/library/yql/core/url_lister/ya.make @@ -11,3 +11,7 @@ PEERDIR( ) END() + +RECURSE( + interface +) diff --git a/ydb/library/yql/core/url_preprocessing/ya.make b/ydb/library/yql/core/url_preprocessing/ya.make index c062ee36f2f..0449776e356 100644 --- a/ydb/library/yql/core/url_preprocessing/ya.make +++ b/ydb/library/yql/core/url_preprocessing/ya.make @@ -14,3 +14,7 @@ PEERDIR( ) END() + +RECURSE( + interface +) diff --git a/ydb/library/yql/core/ya.make b/ydb/library/yql/core/ya.make index ec7b793d991..4fa07131448 100644 --- a/ydb/library/yql/core/ya.make +++ b/ydb/library/yql/core/ya.make @@ -92,4 +92,25 @@ YQL_LAST_ABI_VERSION() END() +RECURSE( + arrow_kernels + cbo + common_opt + credentials + expr_nodes + expr_nodes_gen + extract_predicate + facade + file_storage + issue + peephole_opt + services + spilling + sql_types + type_ann + url_lister + url_preprocessing + user_data +) + RECURSE_FOR_TESTS(ut) diff --git a/ydb/library/yql/minikql/arrow/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/arrow/CMakeLists.darwin-x86_64.txt index 5bb17a45a42..b40197fd49c 100644 --- a/ydb/library/yql/minikql/arrow/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/minikql/arrow/CMakeLists.darwin-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) add_library(yql-minikql-arrow) target_compile_options(yql-minikql-arrow PRIVATE diff --git a/ydb/library/yql/minikql/arrow/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/arrow/CMakeLists.linux-aarch64.txt index 73213515674..38d86148521 100644 --- a/ydb/library/yql/minikql/arrow/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/minikql/arrow/CMakeLists.linux-aarch64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) add_library(yql-minikql-arrow) target_compile_options(yql-minikql-arrow PRIVATE diff --git a/ydb/library/yql/minikql/arrow/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/arrow/CMakeLists.linux-x86_64.txt index 73213515674..38d86148521 100644 --- a/ydb/library/yql/minikql/arrow/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/minikql/arrow/CMakeLists.linux-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) add_library(yql-minikql-arrow) target_compile_options(yql-minikql-arrow PRIVATE diff --git a/ydb/library/yql/minikql/arrow/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/arrow/CMakeLists.windows-x86_64.txt index 5bb17a45a42..b40197fd49c 100644 --- a/ydb/library/yql/minikql/arrow/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/minikql/arrow/CMakeLists.windows-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) add_library(yql-minikql-arrow) target_compile_options(yql-minikql-arrow PRIVATE diff --git a/ydb/library/yql/minikql/arrow/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/arrow/ut/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..245ab8c1813 --- /dev/null +++ b/ydb/library/yql/minikql/arrow/ut/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,81 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-arrow-ut) +target_compile_options(ydb-library-yql-minikql-arrow-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-arrow-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/arrow +) +target_link_libraries(ydb-library-yql-minikql-arrow-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-minikql-arrow + yql-public-udf + udf-service-exception_policy + yql-sql-pg_dummy + minikql-invoke_builtins-llvm +) +target_link_options(ydb-library-yql-minikql-arrow-ut PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-library-yql-minikql-arrow-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/arrow/mkql_functions_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-arrow-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-minikql-arrow-ut + TEST_TARGET + ydb-library-yql-minikql-arrow-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-arrow-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-arrow-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-arrow-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-minikql-arrow-ut + system_allocator +) +vcs_info(ydb-library-yql-minikql-arrow-ut) diff --git a/ydb/library/yql/minikql/arrow/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/arrow/ut/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..719f3857393 --- /dev/null +++ b/ydb/library/yql/minikql/arrow/ut/CMakeLists.linux-aarch64.txt @@ -0,0 +1,84 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-arrow-ut) +target_compile_options(ydb-library-yql-minikql-arrow-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-arrow-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/arrow +) +target_link_libraries(ydb-library-yql-minikql-arrow-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + yql-minikql-arrow + yql-public-udf + udf-service-exception_policy + yql-sql-pg_dummy + minikql-invoke_builtins-llvm +) +target_link_options(ydb-library-yql-minikql-arrow-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-minikql-arrow-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/arrow/mkql_functions_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-arrow-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-minikql-arrow-ut + TEST_TARGET + ydb-library-yql-minikql-arrow-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-arrow-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-arrow-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-arrow-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-minikql-arrow-ut + cpp-malloc-jemalloc +) +vcs_info(ydb-library-yql-minikql-arrow-ut) diff --git a/ydb/library/yql/minikql/arrow/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/arrow/ut/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..d5e533c7bd5 --- /dev/null +++ b/ydb/library/yql/minikql/arrow/ut/CMakeLists.linux-x86_64.txt @@ -0,0 +1,86 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-arrow-ut) +target_compile_options(ydb-library-yql-minikql-arrow-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-arrow-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/arrow +) +target_link_libraries(ydb-library-yql-minikql-arrow-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-minikql-arrow + yql-public-udf + udf-service-exception_policy + yql-sql-pg_dummy + minikql-invoke_builtins-llvm +) +target_link_options(ydb-library-yql-minikql-arrow-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-minikql-arrow-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/arrow/mkql_functions_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-arrow-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-minikql-arrow-ut + TEST_TARGET + ydb-library-yql-minikql-arrow-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-arrow-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-arrow-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-arrow-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-minikql-arrow-ut + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(ydb-library-yql-minikql-arrow-ut) diff --git a/ydb/library/yql/minikql/arrow/ut/CMakeLists.txt b/ydb/library/yql/minikql/arrow/ut/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/minikql/arrow/ut/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/minikql/arrow/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/arrow/ut/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..55cf43ffb11 --- /dev/null +++ b/ydb/library/yql/minikql/arrow/ut/CMakeLists.windows-x86_64.txt @@ -0,0 +1,74 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-arrow-ut) +target_compile_options(ydb-library-yql-minikql-arrow-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-arrow-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/arrow +) +target_link_libraries(ydb-library-yql-minikql-arrow-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-minikql-arrow + yql-public-udf + udf-service-exception_policy + yql-sql-pg_dummy + minikql-invoke_builtins-llvm +) +target_sources(ydb-library-yql-minikql-arrow-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/arrow/mkql_functions_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-arrow-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-minikql-arrow-ut + TEST_TARGET + ydb-library-yql-minikql-arrow-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-arrow-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-arrow-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-arrow-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-minikql-arrow-ut + system_allocator +) +vcs_info(ydb-library-yql-minikql-arrow-ut) diff --git a/ydb/library/yql/minikql/codegen/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/codegen/CMakeLists.darwin-x86_64.txt index cf7af4749a9..377a730d286 100644 --- a/ydb/library/yql/minikql/codegen/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/minikql/codegen/CMakeLists.darwin-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) add_library(yql-minikql-codegen) target_compile_options(yql-minikql-codegen PRIVATE diff --git a/ydb/library/yql/minikql/codegen/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/codegen/CMakeLists.linux-aarch64.txt index 49938a8c462..6a23884af5a 100644 --- a/ydb/library/yql/minikql/codegen/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/minikql/codegen/CMakeLists.linux-aarch64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) add_library(yql-minikql-codegen) target_compile_options(yql-minikql-codegen PRIVATE diff --git a/ydb/library/yql/minikql/codegen/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/codegen/CMakeLists.linux-x86_64.txt index 49938a8c462..6a23884af5a 100644 --- a/ydb/library/yql/minikql/codegen/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/minikql/codegen/CMakeLists.linux-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) add_library(yql-minikql-codegen) target_compile_options(yql-minikql-codegen PRIVATE diff --git a/ydb/library/yql/minikql/codegen/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/codegen/CMakeLists.windows-x86_64.txt index fb5f429c0f1..7fdb97049bc 100644 --- a/ydb/library/yql/minikql/codegen/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/minikql/codegen/CMakeLists.windows-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) add_library(yql-minikql-codegen) target_compile_options(yql-minikql-codegen PRIVATE diff --git a/ydb/library/yql/minikql/codegen/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/codegen/ut/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..fb548f5eeb8 --- /dev/null +++ b/ydb/library/yql/minikql/codegen/ut/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,157 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_rescompiler_bin + TOOL_rescompiler_dependency + tools/rescompiler/bin + rescompiler +) + +add_executable(ydb-library-yql-minikql-codegen-ut) +target_compile_options(ydb-library-yql-minikql-codegen-ut PRIVATE + $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything> +) +target_include_directories(ydb-library-yql-minikql-codegen-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen +) +target_link_libraries(ydb-library-yql-minikql-codegen-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-minikql-codegen + library-cpp-resource +) +target_link_options(ydb-library-yql-minikql-codegen-ut PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-library-yql-minikql-codegen-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/codegen_ut.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/925fda4cf8bf8097eb12d1681cec1fd0.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-codegen-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-minikql-codegen-ut + TEST_TARGET + ydb-library-yql-minikql-codegen-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-codegen-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-codegen-ut + PROPERTY + PROCESSORS + 1 +) +add_custom_command( + OUTPUT + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc + DEPENDS + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc + COMMAND + ${LLVMOPT} + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc + -o + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc + -O2 + -globalopt + -globaldce + -internalize + -internalize-public-api-list=fib#sum_sqr#sum_sqr2#sum_sqr_128#sum_sqr_128_ir#str_size +) +add_custom_command( + OUTPUT + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc + DEPENDS + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.ll.bc + COMMAND + ${LLVMLINK} + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.ll.bc + -o + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +resources(ydb-library-yql-minikql-codegen-ut + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/925fda4cf8bf8097eb12d1681cec1fd0.cpp + INPUTS + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc + KEYS + /llvm_bc/Funcs +) +target_allocator(ydb-library-yql-minikql-codegen-ut + system_allocator +) +vcs_info(ydb-library-yql-minikql-codegen-ut) diff --git a/ydb/library/yql/minikql/codegen/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/codegen/ut/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..32a6487c08f --- /dev/null +++ b/ydb/library/yql/minikql/codegen/ut/CMakeLists.linux-aarch64.txt @@ -0,0 +1,160 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_rescompiler_bin + TOOL_rescompiler_dependency + tools/rescompiler/bin + rescompiler +) + +add_executable(ydb-library-yql-minikql-codegen-ut) +target_compile_options(ydb-library-yql-minikql-codegen-ut PRIVATE + $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything> +) +target_include_directories(ydb-library-yql-minikql-codegen-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen +) +target_link_libraries(ydb-library-yql-minikql-codegen-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + yql-minikql-codegen + library-cpp-resource +) +target_link_options(ydb-library-yql-minikql-codegen-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-minikql-codegen-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/codegen_ut.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/925fda4cf8bf8097eb12d1681cec1fd0.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-codegen-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-minikql-codegen-ut + TEST_TARGET + ydb-library-yql-minikql-codegen-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-codegen-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-codegen-ut + PROPERTY + PROCESSORS + 1 +) +add_custom_command( + OUTPUT + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc + DEPENDS + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc + COMMAND + ${LLVMOPT} + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc + -o + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc + -O2 + -globalopt + -globaldce + -internalize + -internalize-public-api-list=fib#sum_sqr#sum_sqr2#sum_sqr_128#sum_sqr_128_ir#str_size +) +add_custom_command( + OUTPUT + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc + DEPENDS + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.ll.bc + COMMAND + ${LLVMLINK} + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.ll.bc + -o + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +resources(ydb-library-yql-minikql-codegen-ut + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/925fda4cf8bf8097eb12d1681cec1fd0.cpp + INPUTS + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc + KEYS + /llvm_bc/Funcs +) +target_allocator(ydb-library-yql-minikql-codegen-ut + cpp-malloc-jemalloc +) +vcs_info(ydb-library-yql-minikql-codegen-ut) diff --git a/ydb/library/yql/minikql/codegen/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/codegen/ut/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..d3a3bee4e27 --- /dev/null +++ b/ydb/library/yql/minikql/codegen/ut/CMakeLists.linux-x86_64.txt @@ -0,0 +1,162 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_rescompiler_bin + TOOL_rescompiler_dependency + tools/rescompiler/bin + rescompiler +) + +add_executable(ydb-library-yql-minikql-codegen-ut) +target_compile_options(ydb-library-yql-minikql-codegen-ut PRIVATE + $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything> +) +target_include_directories(ydb-library-yql-minikql-codegen-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen +) +target_link_libraries(ydb-library-yql-minikql-codegen-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-minikql-codegen + library-cpp-resource +) +target_link_options(ydb-library-yql-minikql-codegen-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-minikql-codegen-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/codegen_ut.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/925fda4cf8bf8097eb12d1681cec1fd0.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-codegen-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-minikql-codegen-ut + TEST_TARGET + ydb-library-yql-minikql-codegen-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-codegen-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-codegen-ut + PROPERTY + PROCESSORS + 1 +) +add_custom_command( + OUTPUT + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc + DEPENDS + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc + COMMAND + ${LLVMOPT} + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc + -o + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc + -O2 + -globalopt + -globaldce + -internalize + -internalize-public-api-list=fib#sum_sqr#sum_sqr2#sum_sqr_128#sum_sqr_128_ir#str_size +) +add_custom_command( + OUTPUT + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc + DEPENDS + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.ll.bc + COMMAND + ${LLVMLINK} + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.ll.bc + -o + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +resources(ydb-library-yql-minikql-codegen-ut + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/925fda4cf8bf8097eb12d1681cec1fd0.cpp + INPUTS + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc + KEYS + /llvm_bc/Funcs +) +target_allocator(ydb-library-yql-minikql-codegen-ut + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(ydb-library-yql-minikql-codegen-ut) diff --git a/ydb/library/yql/minikql/codegen/ut/CMakeLists.txt b/ydb/library/yql/minikql/codegen/ut/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/minikql/codegen/ut/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/minikql/codegen/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/codegen/ut/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..2524c3f4efd --- /dev/null +++ b/ydb/library/yql/minikql/codegen/ut/CMakeLists.windows-x86_64.txt @@ -0,0 +1,150 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_rescompiler_bin + TOOL_rescompiler_dependency + tools/rescompiler/bin + rescompiler +) + +add_executable(ydb-library-yql-minikql-codegen-ut) +target_compile_options(ydb-library-yql-minikql-codegen-ut PRIVATE + $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything> +) +target_include_directories(ydb-library-yql-minikql-codegen-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen +) +target_link_libraries(ydb-library-yql-minikql-codegen-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-minikql-codegen + library-cpp-resource +) +target_sources(ydb-library-yql-minikql-codegen-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/codegen_ut.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/925fda4cf8bf8097eb12d1681cec1fd0.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-codegen-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-minikql-codegen-ut + TEST_TARGET + ydb-library-yql-minikql-codegen-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-codegen-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-codegen-ut + PROPERTY + PROCESSORS + 1 +) +add_custom_command( + OUTPUT + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc + DEPENDS + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc + COMMAND + ${LLVMOPT} + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc + -o + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc + -O2 + -globalopt + -globaldce + -internalize + -internalize-public-api-list=fib#sum_sqr#sum_sqr2#sum_sqr_128#sum_sqr_128_ir#str_size +) +add_custom_command( + OUTPUT + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc + DEPENDS + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit_win.ll.bc + COMMAND + ${LLVMLINK} + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit_win.ll.bc + -o + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc + ${CLANGPLUSPLUS} + -Wno-unknown-warning-option + -emit-llvm +) +resources(ydb-library-yql-minikql-codegen-ut + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/925fda4cf8bf8097eb12d1681cec1fd0.cpp + INPUTS + ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc + KEYS + /llvm_bc/Funcs +) +target_allocator(ydb-library-yql-minikql-codegen-ut + system_allocator +) +vcs_info(ydb-library-yql-minikql-codegen-ut) diff --git a/ydb/library/yql/minikql/comp_nodes/CMakeLists.txt b/ydb/library/yql/minikql/comp_nodes/CMakeLists.txt index e519707fa41..4716de1ef37 100644 --- a/ydb/library/yql/minikql/comp_nodes/CMakeLists.txt +++ b/ydb/library/yql/minikql/comp_nodes/CMakeLists.txt @@ -7,3 +7,5 @@ add_subdirectory(llvm) +add_subdirectory(no_llvm) +add_subdirectory(ut) diff --git a/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..f10e5ccc395 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,157 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(minikql-comp_nodes-no_llvm) +target_compile_options(minikql-comp_nodes-no_llvm PRIVATE + -mprfchw + -DMKQL_DISABLE_CODEGEN + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(minikql-comp_nodes-no_llvm PUBLIC + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub +) +target_link_libraries(minikql-comp_nodes-no_llvm PUBLIC + contrib-libs-cxxsupp + yutil + libs-apache-arrow + ydb-library-binary_json + library-yql-minikql + yql-minikql-arrow + public-udf-arrow + parser-pg_wrapper-interface + library-yql-utils + cpp-actors-core + minikql-invoke_builtins-no_llvm +) +target_sources(minikql-comp_nodes-no_llvm PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_addmember.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_aggrcount.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_append.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_apply.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_count.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_factory.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_minmax.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_some.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_sum.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_coalesce.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_extend.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_if.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_just.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_logical.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_compress.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_func.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_skiptake.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_top.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_tuple.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_blocks.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_callable.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chain_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chain1_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_check_args.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chopper.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_coalesce.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_collect.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_contains.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_div.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_mod.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_mul.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_dictitems.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_discard.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_element.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_ensure.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_enumerate.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_exists.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_extend.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_filter.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_flow.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fold.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fold1.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_frombytes.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fromstring.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fromyson.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_group.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_grace_join.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_grace_join_imp.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_guess.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_hasitems.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_heap.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_if.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_ifpresent.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_invoke.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_iterable.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_iterator.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_join_dict.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_lazy_list.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_length.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_listfromrange.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_llvm_base.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_logical.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_lookup.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_mapnext.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_map_join.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_multimap.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_next_value.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_nop.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_now.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_null.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_pickle.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_prepend.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_queue.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_random.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_reduce.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_removemember.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_replicate.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_reverse.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_rh_hash.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_round.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_safe_circular_buffer.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_scalar_apply.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_seq.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_size.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_skip.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_sort.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_source.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_state.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_to_list.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_condense.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_condense1.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_switch.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_take.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_timezone.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tobytes.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_toindexdict.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tooptional.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tostring.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_udf.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_unwrap.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_varitem.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_visitall.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_way.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_weakmember.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_while.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_chain_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_chopper.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_combine.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_condense.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_filter.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_top_sort.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_withcontext.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_zip.cpp +) diff --git a/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..96b0c048ea6 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.linux-aarch64.txt @@ -0,0 +1,158 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(minikql-comp_nodes-no_llvm) +target_compile_options(minikql-comp_nodes-no_llvm PRIVATE + -mprfchw + -DMKQL_DISABLE_CODEGEN + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(minikql-comp_nodes-no_llvm PUBLIC + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub +) +target_link_libraries(minikql-comp_nodes-no_llvm PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + libs-apache-arrow + ydb-library-binary_json + library-yql-minikql + yql-minikql-arrow + public-udf-arrow + parser-pg_wrapper-interface + library-yql-utils + cpp-actors-core + minikql-invoke_builtins-no_llvm +) +target_sources(minikql-comp_nodes-no_llvm PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_addmember.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_aggrcount.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_append.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_apply.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_count.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_factory.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_minmax.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_some.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_sum.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_coalesce.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_extend.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_if.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_just.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_logical.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_compress.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_func.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_skiptake.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_top.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_tuple.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_blocks.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_callable.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chain_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chain1_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_check_args.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chopper.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_coalesce.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_collect.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_contains.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_div.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_mod.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_mul.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_dictitems.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_discard.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_element.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_ensure.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_enumerate.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_exists.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_extend.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_filter.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_flow.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fold.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fold1.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_frombytes.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fromstring.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fromyson.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_group.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_grace_join.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_grace_join_imp.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_guess.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_hasitems.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_heap.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_if.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_ifpresent.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_invoke.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_iterable.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_iterator.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_join_dict.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_lazy_list.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_length.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_listfromrange.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_llvm_base.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_logical.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_lookup.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_mapnext.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_map_join.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_multimap.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_next_value.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_nop.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_now.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_null.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_pickle.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_prepend.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_queue.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_random.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_reduce.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_removemember.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_replicate.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_reverse.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_rh_hash.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_round.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_safe_circular_buffer.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_scalar_apply.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_seq.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_size.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_skip.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_sort.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_source.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_state.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_to_list.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_condense.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_condense1.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_switch.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_take.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_timezone.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tobytes.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_toindexdict.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tooptional.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tostring.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_udf.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_unwrap.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_varitem.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_visitall.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_way.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_weakmember.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_while.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_chain_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_chopper.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_combine.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_condense.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_filter.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_top_sort.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_withcontext.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_zip.cpp +) diff --git a/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..96b0c048ea6 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.linux-x86_64.txt @@ -0,0 +1,158 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(minikql-comp_nodes-no_llvm) +target_compile_options(minikql-comp_nodes-no_llvm PRIVATE + -mprfchw + -DMKQL_DISABLE_CODEGEN + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(minikql-comp_nodes-no_llvm PUBLIC + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub +) +target_link_libraries(minikql-comp_nodes-no_llvm PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + libs-apache-arrow + ydb-library-binary_json + library-yql-minikql + yql-minikql-arrow + public-udf-arrow + parser-pg_wrapper-interface + library-yql-utils + cpp-actors-core + minikql-invoke_builtins-no_llvm +) +target_sources(minikql-comp_nodes-no_llvm PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_addmember.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_aggrcount.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_append.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_apply.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_count.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_factory.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_minmax.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_some.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_sum.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_coalesce.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_extend.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_if.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_just.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_logical.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_compress.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_func.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_skiptake.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_top.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_tuple.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_blocks.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_callable.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chain_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chain1_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_check_args.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chopper.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_coalesce.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_collect.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_contains.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_div.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_mod.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_mul.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_dictitems.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_discard.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_element.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_ensure.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_enumerate.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_exists.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_extend.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_filter.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_flow.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fold.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fold1.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_frombytes.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fromstring.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fromyson.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_group.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_grace_join.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_grace_join_imp.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_guess.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_hasitems.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_heap.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_if.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_ifpresent.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_invoke.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_iterable.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_iterator.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_join_dict.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_lazy_list.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_length.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_listfromrange.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_llvm_base.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_logical.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_lookup.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_mapnext.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_map_join.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_multimap.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_next_value.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_nop.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_now.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_null.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_pickle.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_prepend.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_queue.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_random.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_reduce.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_removemember.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_replicate.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_reverse.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_rh_hash.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_round.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_safe_circular_buffer.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_scalar_apply.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_seq.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_size.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_skip.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_sort.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_source.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_state.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_to_list.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_condense.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_condense1.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_switch.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_take.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_timezone.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tobytes.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_toindexdict.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tooptional.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tostring.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_udf.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_unwrap.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_varitem.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_visitall.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_way.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_weakmember.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_while.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_chain_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_chopper.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_combine.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_condense.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_filter.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_top_sort.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_withcontext.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_zip.cpp +) diff --git a/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.txt b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..f10e5ccc395 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.windows-x86_64.txt @@ -0,0 +1,157 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(minikql-comp_nodes-no_llvm) +target_compile_options(minikql-comp_nodes-no_llvm PRIVATE + -mprfchw + -DMKQL_DISABLE_CODEGEN + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(minikql-comp_nodes-no_llvm PUBLIC + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub +) +target_link_libraries(minikql-comp_nodes-no_llvm PUBLIC + contrib-libs-cxxsupp + yutil + libs-apache-arrow + ydb-library-binary_json + library-yql-minikql + yql-minikql-arrow + public-udf-arrow + parser-pg_wrapper-interface + library-yql-utils + cpp-actors-core + minikql-invoke_builtins-no_llvm +) +target_sources(minikql-comp_nodes-no_llvm PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_addmember.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_aggrcount.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_append.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_apply.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_count.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_factory.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_minmax.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_some.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_sum.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_coalesce.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_extend.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_if.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_just.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_logical.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_compress.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_func.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_skiptake.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_top.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_tuple.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_blocks.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_callable.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chain_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chain1_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_check_args.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chopper.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_coalesce.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_collect.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_contains.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_div.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_mod.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_mul.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_dictitems.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_discard.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_element.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_ensure.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_enumerate.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_exists.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_extend.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_filter.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_flow.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fold.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fold1.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_frombytes.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fromstring.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fromyson.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_group.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_grace_join.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_grace_join_imp.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_guess.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_hasitems.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_heap.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_if.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_ifpresent.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_invoke.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_iterable.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_iterator.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_join_dict.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_lazy_list.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_length.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_listfromrange.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_llvm_base.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_logical.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_lookup.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_mapnext.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_map_join.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_multimap.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_next_value.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_nop.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_now.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_null.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_pickle.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_prepend.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_queue.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_random.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_reduce.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_removemember.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_replicate.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_reverse.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_rh_hash.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_round.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_safe_circular_buffer.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_scalar_apply.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_seq.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_size.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_skip.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_sort.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_source.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_state.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_to_list.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_condense.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_condense1.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_switch.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_take.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_timezone.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tobytes.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_toindexdict.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tooptional.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tostring.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_udf.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_unwrap.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_varitem.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_visitall.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_way.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_weakmember.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_while.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_chain_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_chopper.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_combine.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_condense.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_filter.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_map.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_top_sort.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_withcontext.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_zip.cpp +) diff --git a/ydb/library/yql/minikql/comp_nodes/no_llvm/ya.make b/ydb/library/yql/minikql/comp_nodes/no_llvm/ya.make new file mode 100644 index 00000000000..589a87f3541 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/no_llvm/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +CXXFLAGS(-DMKQL_DISABLE_CODEGEN) + +ADDINCL(GLOBAL ydb/library/yql/minikql/codegen/llvm_stub) + +INCLUDE(../ya.make.inc) + +PEERDIR(ydb/library/yql/minikql/invoke_builtins/no_llvm) + +END() diff --git a/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..4cc7327d09f --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,124 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-comp_nodes-ut) +target_compile_options(ydb-library-yql-minikql-comp_nodes-ut PRIVATE + -mprfchw + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-comp_nodes-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/llvm +) +target_link_libraries(ydb-library-yql-minikql-comp_nodes-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + minikql-comp_nodes-llvm + yql-public-udf + public-udf-arrow + udf-service-exception_policy + yql-sql-pg_dummy +) +target_link_options(ydb-library-yql-minikql-comp_nodes-ut PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-library-yql-minikql-comp_nodes-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_test_factory.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_bit_utils_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_compress_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_skiptake_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_blocks_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_condense_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_decimal_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_chain_map_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_filters_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_flatmap_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_fromstring_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multimap_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_fold_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_heap_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_compare_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_dict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_grace_join_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_map_join_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_safe_circular_buffer_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_sort_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_switch_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_variant_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chain_map_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chopper_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_condense_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_filter_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_map_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_nodes_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_stream_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_top_sort_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_listfromrange_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_mapnext_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_rh_hash_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-comp_nodes-ut + PROPERTY + SPLIT_FACTOR + 60 +) +add_yunittest( + NAME + ydb-library-yql-minikql-comp_nodes-ut + TEST_TARGET + ydb-library-yql-minikql-comp_nodes-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-comp_nodes-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-comp_nodes-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-comp_nodes-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-minikql-comp_nodes-ut + system_allocator +) +vcs_info(ydb-library-yql-minikql-comp_nodes-ut) diff --git a/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..62b305769ae --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.linux-aarch64.txt @@ -0,0 +1,127 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-comp_nodes-ut) +target_compile_options(ydb-library-yql-minikql-comp_nodes-ut PRIVATE + -mprfchw + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-comp_nodes-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/llvm +) +target_link_libraries(ydb-library-yql-minikql-comp_nodes-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + minikql-comp_nodes-llvm + yql-public-udf + public-udf-arrow + udf-service-exception_policy + yql-sql-pg_dummy +) +target_link_options(ydb-library-yql-minikql-comp_nodes-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-minikql-comp_nodes-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_test_factory.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_bit_utils_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_compress_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_skiptake_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_blocks_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_condense_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_decimal_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_chain_map_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_filters_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_flatmap_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_fromstring_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multimap_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_fold_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_heap_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_compare_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_dict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_grace_join_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_map_join_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_safe_circular_buffer_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_sort_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_switch_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_variant_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chain_map_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chopper_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_condense_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_filter_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_map_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_nodes_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_stream_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_top_sort_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_listfromrange_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_mapnext_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_rh_hash_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-comp_nodes-ut + PROPERTY + SPLIT_FACTOR + 60 +) +add_yunittest( + NAME + ydb-library-yql-minikql-comp_nodes-ut + TEST_TARGET + ydb-library-yql-minikql-comp_nodes-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-comp_nodes-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-comp_nodes-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-comp_nodes-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-minikql-comp_nodes-ut + cpp-malloc-jemalloc +) +vcs_info(ydb-library-yql-minikql-comp_nodes-ut) diff --git a/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..795dc42da2a --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.linux-x86_64.txt @@ -0,0 +1,129 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-comp_nodes-ut) +target_compile_options(ydb-library-yql-minikql-comp_nodes-ut PRIVATE + -mprfchw + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-comp_nodes-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/llvm +) +target_link_libraries(ydb-library-yql-minikql-comp_nodes-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + minikql-comp_nodes-llvm + yql-public-udf + public-udf-arrow + udf-service-exception_policy + yql-sql-pg_dummy +) +target_link_options(ydb-library-yql-minikql-comp_nodes-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-minikql-comp_nodes-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_test_factory.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_bit_utils_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_compress_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_skiptake_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_blocks_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_condense_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_decimal_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_chain_map_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_filters_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_flatmap_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_fromstring_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multimap_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_fold_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_heap_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_compare_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_dict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_grace_join_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_map_join_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_safe_circular_buffer_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_sort_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_switch_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_variant_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chain_map_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chopper_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_condense_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_filter_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_map_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_nodes_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_stream_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_top_sort_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_listfromrange_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_mapnext_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_rh_hash_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-comp_nodes-ut + PROPERTY + SPLIT_FACTOR + 60 +) +add_yunittest( + NAME + ydb-library-yql-minikql-comp_nodes-ut + TEST_TARGET + ydb-library-yql-minikql-comp_nodes-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-comp_nodes-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-comp_nodes-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-comp_nodes-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-minikql-comp_nodes-ut + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(ydb-library-yql-minikql-comp_nodes-ut) diff --git a/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.txt b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..6a530932fed --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.windows-x86_64.txt @@ -0,0 +1,117 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-comp_nodes-ut) +target_compile_options(ydb-library-yql-minikql-comp_nodes-ut PRIVATE + -mprfchw + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-comp_nodes-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/llvm +) +target_link_libraries(ydb-library-yql-minikql-comp_nodes-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + minikql-comp_nodes-llvm + yql-public-udf + public-udf-arrow + udf-service-exception_policy + yql-sql-pg_dummy +) +target_sources(ydb-library-yql-minikql-comp_nodes-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_test_factory.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_bit_utils_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_compress_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_skiptake_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_blocks_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_condense_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_decimal_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_chain_map_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_filters_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_flatmap_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_fromstring_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multimap_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_fold_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_heap_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_compare_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_dict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_grace_join_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_map_join_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_safe_circular_buffer_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_sort_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_switch_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_variant_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chain_map_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chopper_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_condense_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_filter_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_map_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_nodes_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_stream_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_top_sort_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_listfromrange_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_mapnext_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_rh_hash_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-comp_nodes-ut + PROPERTY + SPLIT_FACTOR + 60 +) +add_yunittest( + NAME + ydb-library-yql-minikql-comp_nodes-ut + TEST_TARGET + ydb-library-yql-minikql-comp_nodes-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-comp_nodes-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-comp_nodes-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-comp_nodes-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-minikql-comp_nodes-ut + system_allocator +) +vcs_info(ydb-library-yql-minikql-comp_nodes-ut) diff --git a/ydb/library/yql/minikql/comp_nodes/ut/build_no_codegen.sh b/ydb/library/yql/minikql/comp_nodes/ut/build_no_codegen.sh new file mode 100755 index 00000000000..9ad6ea4eabe --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/build_no_codegen.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -ex + +ya make -D MKQL_DISABLE_CODEGEN --target-platform=DEFAULT-LINUX-X86_64 --target-platform-flag=CFLAGS='-DMKQL_DISABLE_CODEGEN' diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_bit_utils_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_bit_utils_ut.cpp new file mode 100644 index 00000000000..619de191ea9 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_bit_utils_ut.cpp @@ -0,0 +1,67 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/arrow/mkql_bit_utils.h> + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + +ui8 NaiveCompressByte(ui8 value, ui8 mask) { + ui8 result = 0; + ui8 outPos = 0; + for (ui8 i = 0; i < 8; ++i) { + if (mask & (1 << i)) { + ui8 bit = (value & (1 << i)) != 0; + result |= (bit << outPos); + ++outPos; + } + } + return result; +} + +} // namespace + + +Y_UNIT_TEST_SUITE(TMiniKQLBitUtilsTest) { +Y_UNIT_TEST(TestCompressByte) { + for (size_t value = 0; value < 256; ++value) { + for (size_t mask = 0; mask < 256; ++mask) { + UNIT_ASSERT_EQUAL(NaiveCompressByte(value, mask), CompressByte(value, mask)); + } + } +} + +Y_UNIT_TEST(TestLoad) { + const ui8 src[] = {0b01110100, 0b11011101, 0b01101011}; + UNIT_ASSERT_EQUAL(LoadByteUnaligned(src, 10), 0b11110111); + UNIT_ASSERT_EQUAL(LoadByteUnaligned(src, 16), 0b01101011); +} + +Y_UNIT_TEST(CompressAligned) { + const ui8 data[] = {0b01110100, 0b11011101, 0b01101011}; + const ui8 mask[] = {0b11101100, 0b10111010, 0b10001111}; + ui8 result[100]; + auto res = CompressBitmap(data, 0, mask, 0, result, 0, 24); + UNIT_ASSERT_EQUAL(res, 15); + UNIT_ASSERT_EQUAL(result[0], 0b11001101); + UNIT_ASSERT_EQUAL(result[1] & 0x7fu, 0b00101110); +} + +Y_UNIT_TEST(CompressUnalignedOutput) { + const ui8 data[] = {0b01110100, 0b11011101, 0b01101011}; + const ui8 mask[] = {0b11101100, 0b10111010, 0b10001111}; + ui8 result[100]; + result[0] = 0b101; + auto res = CompressBitmap(data, 0, mask, 0, result, 3, 24); + UNIT_ASSERT_EQUAL(res, 18); + UNIT_ASSERT_EQUAL(result[0], 0b01101101); + UNIT_ASSERT_EQUAL(result[1], 0b01110110); + UNIT_ASSERT_EQUAL(result[2] & 0x3, 0b01); +} + +} + +} // namespace NMiniKQL +} // namespace NKikimr + diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_compress_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_compress_ut.cpp new file mode 100644 index 00000000000..f68b44d7705 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_compress_ut.cpp @@ -0,0 +1,227 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> +#include <ydb/library/yql/minikql/computation/mkql_block_builder.h> + +#include <util/random/random.h> + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + +void DoNestedTuplesCompressTest(bool useRandom, bool doFilter) { + TSetup<false> setup; + auto& pb = *setup.PgmBuilder; + + const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id); + const auto boolType = pb.NewDataType(NUdf::TDataType<bool>::Id); + const auto utf8Type = pb.NewDataType(NUdf::EDataSlot::Utf8); + + const auto innerTupleType = pb.NewTupleType({ui64Type, boolType, utf8Type}); + const auto outerTupleType = pb.NewTupleType({ui64Type, innerTupleType, utf8Type}); + const auto finalTupleType = pb.NewTupleType({ui64Type, outerTupleType, boolType}); + + const auto resultTupleType = pb.NewTupleType({ui64Type, outerTupleType}); + + TVector<TRuntimeNode> items; + static_assert(MaxBlockSizeInBytes % 4 == 0); + constexpr size_t fixedStrSize = MaxBlockSizeInBytes / 4; + + if (useRandom) { + SetRandomSeed(0); + } + + for (size_t i = 0; i < 95; ++i) { + std::string str; + bool filterValue; + if (useRandom) { + size_t len = RandomNumber<size_t>(2 * MaxBlockSizeInBytes); + str.reserve(len); + for (size_t i = 0; i < len; ++i) { + str.push_back((char)RandomNumber<ui8>(128)); + } + if (doFilter) { + filterValue = RandomNumber<ui8>() & 1; + } else { + filterValue = true; + } + } else { + str = std::string(fixedStrSize, ' ' + i); + if (doFilter) { + filterValue = (i % 4) < 2; + } else { + filterValue = true; + } + } + + auto innerTuple = pb.NewTuple(innerTupleType, { + pb.NewDataLiteral<ui64>(i), + pb.NewDataLiteral<bool>(i % 2), + pb.NewDataLiteral<NUdf::EDataSlot::Utf8>((i % 2) ? str : std::string()), + }); + auto outerTuple = pb.NewTuple(outerTupleType, { + pb.NewDataLiteral<ui64>(i), + innerTuple, + pb.NewDataLiteral<NUdf::EDataSlot::Utf8>((i % 2) ? std::string() : str), + }); + + auto finalTuple = pb.NewTuple(finalTupleType, { + pb.NewDataLiteral<ui64>(i), + outerTuple, + pb.NewDataLiteral<bool>(filterValue), + }); + items.push_back(finalTuple); + } + + const auto list = pb.NewList(finalTupleType, std::move(items)); + + auto node = pb.ToFlow(list); + node = pb.ExpandMap(node, [&](TRuntimeNode item) -> TRuntimeNode::TList { + return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; + }); + node = pb.WideToBlocks(node); + + node = pb.BlockExpandChunked(node); + node = pb.WideSkipBlocks(node, pb.NewDataLiteral<ui64>(19)); + node = pb.BlockCompress(node, 2); + node = pb.WideFromBlocks(node); + + node = pb.NarrowMap(node, [&](TRuntimeNode::TList items) -> TRuntimeNode { + return pb.NewTuple(resultTupleType, {items[0], items[1]}); + }); + + const auto pgmReturn = pb.ForwardList(node); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + + if (useRandom) { + SetRandomSeed(0); + } + + for (size_t i = 0; i < 95; ++i) { + std::string str; + bool filterValue; + if (useRandom) { + size_t len = RandomNumber<size_t>(2 * MaxBlockSizeInBytes); + str.reserve(len); + for (size_t i = 0; i < len; ++i) { + str.push_back((char)RandomNumber<ui8>(128)); + } + if (doFilter) { + filterValue = RandomNumber<ui8>() & 1; + } else { + filterValue = true; + } + } else { + str = std::string(fixedStrSize, ' ' + i); + if (doFilter) { + filterValue = (i % 4) < 2; + } else { + filterValue = true; + } + } + + if (i < 19 || !filterValue) { + continue; + } + + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + ui64 topNum = item.GetElement(0).Get<ui64>(); + const auto& outer = item.GetElement(1); + + ui64 num = outer.GetElement(0).Get<ui64>(); + const auto& inner = outer.GetElement(1); + + auto outerStrVal = outer.GetElement(2); + std::string_view outerStr = outerStrVal.AsStringRef(); + + ui64 innerNum = inner.GetElement(0).Get<ui64>(); + bool innerBool = inner.GetElement(1).Get<bool>(); + auto innerStrVal = inner.GetElement(2); + + std::string_view innerStr = innerStrVal.AsStringRef(); + + UNIT_ASSERT_VALUES_EQUAL(num, i); + UNIT_ASSERT_VALUES_EQUAL(topNum, i); + UNIT_ASSERT_VALUES_EQUAL(innerNum, i); + UNIT_ASSERT_VALUES_EQUAL(innerBool, i % 2); + + std::string expectedInner = (i % 2) ? str : std::string(); + std::string expectedOuter = (i % 2) ? std::string() : str; + + UNIT_ASSERT(innerStr == expectedInner); + UNIT_ASSERT(outerStr == expectedOuter); + } + + NUdf::TUnboxedValue item; + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); +} + +} //namespace + +Y_UNIT_TEST_SUITE(TMiniKQLBlockCompressTest) { +Y_UNIT_TEST(CompressBasic) { + TSetup<false> setup; + auto& pb = *setup.PgmBuilder; + + const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id); + const auto boolType = pb.NewDataType(NUdf::TDataType<bool>::Id); + const auto tupleType = pb.NewTupleType({boolType, ui64Type, boolType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewDataLiteral<bool>(false), pb.NewDataLiteral<ui64>(1), pb.NewDataLiteral<bool>(true)}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewDataLiteral<bool>(true), pb.NewDataLiteral<ui64>(2), pb.NewDataLiteral<bool>(false)}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewDataLiteral<bool>(false), pb.NewDataLiteral<ui64>(3), pb.NewDataLiteral<bool>(true)}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewDataLiteral<bool>(false), pb.NewDataLiteral<ui64>(4), pb.NewDataLiteral<bool>(true)}); + const auto data5 = pb.NewTuple(tupleType, {pb.NewDataLiteral<bool>(true), pb.NewDataLiteral<ui64>(5), pb.NewDataLiteral<bool>(false)}); + const auto data6 = pb.NewTuple(tupleType, {pb.NewDataLiteral<bool>(true), pb.NewDataLiteral<ui64>(6), pb.NewDataLiteral<bool>(true)}); + const auto data7 = pb.NewTuple(tupleType, {pb.NewDataLiteral<bool>(false), pb.NewDataLiteral<ui64>(7), pb.NewDataLiteral<bool>(true)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7}); + const auto flow = pb.ToFlow(list); + + const auto wideFlow = pb.ExpandMap(flow, [&](TRuntimeNode item) -> TRuntimeNode::TList { + return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; + }); + const auto compressedFlow = pb.WideFromBlocks(pb.BlockCompress(pb.WideToBlocks(wideFlow), 0)); + const auto narrowFlow = pb.NarrowMap(compressedFlow, [&](TRuntimeNode::TList items) -> TRuntimeNode { + return pb.NewTuple({items[0], items[1]}); + }); + + const auto pgmReturn = pb.ForwardList(narrowFlow); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<ui64>(), 2); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<bool>(), false); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<ui64>(), 5); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<bool>(), false); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<ui64>(), 6); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<bool>(), true); + + UNIT_ASSERT(!iterator.Next(item)); +} + +Y_UNIT_TEST(CompressNestedTuples) { + DoNestedTuplesCompressTest(false, true); + DoNestedTuplesCompressTest(false, false); +} + +Y_UNIT_TEST(CompressNestedTuplesWithRandom) { + DoNestedTuplesCompressTest(true, true); + DoNestedTuplesCompressTest(true, false); +} + +} + +} // namespace NMiniKQL +} // namespace NKikimr diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_skiptake_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_skiptake_ut.cpp new file mode 100644 index 00000000000..830ee7152ad --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_skiptake_ut.cpp @@ -0,0 +1,179 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/arrow/arrow_defs.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> + +#include <arrow/array/builder_primitive.h> + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + +class TTestBlockFlowWrapper: public TStatefulWideFlowComputationNode<TTestBlockFlowWrapper> { + typedef TStatefulWideFlowComputationNode<TTestBlockFlowWrapper> TBaseComputation; + +public: + TTestBlockFlowWrapper(TComputationMutables& mutables, size_t blockSize, size_t blockCount) + : TBaseComputation(mutables, nullptr, EValueRepresentation::Any) + , BlockSize(blockSize) + , BlockCount(blockCount) + { + } + + EFetchResult DoCalculate(NUdf::TUnboxedValue& state, TComputationContext& ctx, NUdf::TUnboxedValue*const* output) const { + if (!state.HasValue()) { + state = NUdf::TUnboxedValue::Zero(); + } + + ui64 index = state.Get<ui64>(); + if (index >= BlockCount) { + return EFetchResult::Finish; + } + + arrow::UInt64Builder builder(&ctx.ArrowMemoryPool); + ARROW_OK(builder.Reserve(BlockSize)); + for (size_t i = 0; i < BlockSize; ++i) { + builder.UnsafeAppend(index * BlockSize + i); + } + + std::shared_ptr<arrow::ArrayData> block; + ARROW_OK(builder.FinishInternal(&block)); + + *output[0] = ctx.HolderFactory.CreateArrowBlock(std::move(block)); + *output[1] = ctx.HolderFactory.CreateArrowBlock(arrow::Datum(std::make_shared<arrow::UInt64Scalar>(index))); + *output[2] = ctx.HolderFactory.CreateArrowBlock(arrow::Datum(std::make_shared<arrow::UInt64Scalar>(BlockSize))); + + state = NUdf::TUnboxedValuePod(++index); + return EFetchResult::One; + } + +private: + void RegisterDependencies() const final { + } + + const size_t BlockSize; + const size_t BlockCount; +}; + +IComputationNode* WrapTestBlockFlow(TCallable& callable, const TComputationNodeFactoryContext& ctx) { + MKQL_ENSURE(callable.GetInputsCount() == 0, "Expected no args"); + return new TTestBlockFlowWrapper(ctx.Mutables, 5, 2); +} + +TIntrusivePtr<IRandomProvider> CreateRandomProvider() { + return CreateDeterministicRandomProvider(1); +} + +TIntrusivePtr<ITimeProvider> CreateTimeProvider() { + return CreateDeterministicTimeProvider(10000000); +} + +TComputationNodeFactory GetTestFactory() { + return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* { + if (callable.GetType()->GetName() == "TestBlockFlow") { + return WrapTestBlockFlow(callable, ctx); + } + return GetBuiltinFactory()(callable, ctx); + }; +} + +struct TSetup_ { + TSetup_() + : Alloc(__LOCATION__) + { + FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); + RandomProvider = CreateRandomProvider(); + TimeProvider = CreateTimeProvider(); + + Env.Reset(new TTypeEnvironment(Alloc)); + PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry)); + } + + TAutoPtr<IComputationGraph> BuildGraph(TRuntimeNode pgm, EGraphPerProcess graphPerProcess = EGraphPerProcess::Multi, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) { + Explorer.Walk(pgm.GetNode(), *Env); + TComputationPatternOpts opts(Alloc.Ref(), *Env, GetTestFactory(), FunctionRegistry.Get(), + NUdf::EValidateMode::None, NUdf::EValidatePolicy::Exception, "OFF", graphPerProcess); + Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts); + return Pattern->Clone(opts.ToComputationOptions(*RandomProvider, *TimeProvider)); + } + + TIntrusivePtr<IFunctionRegistry> FunctionRegistry; + TIntrusivePtr<IRandomProvider> RandomProvider; + TIntrusivePtr<ITimeProvider> TimeProvider; + + TScopedAlloc Alloc; + THolder<TTypeEnvironment> Env; + THolder<TProgramBuilder> PgmBuilder; + + TExploringNodeVisitor Explorer; + IComputationPattern::TPtr Pattern; +}; + +TRuntimeNode MakeFlow(TSetup_& setup) { + TProgramBuilder& pb = *setup.PgmBuilder; + TCallableBuilder callableBuilder(*setup.Env, "TestBlockFlow", + pb.NewFlowType( + pb.NewMultiType({ + pb.NewBlockType(pb.NewDataType(NUdf::EDataSlot::Uint64), TBlockType::EShape::Many), + pb.NewBlockType(pb.NewDataType(NUdf::EDataSlot::Uint64), TBlockType::EShape::Scalar), + pb.NewBlockType(pb.NewDataType(NUdf::EDataSlot::Uint64), TBlockType::EShape::Scalar), + }))); + return TRuntimeNode(callableBuilder.Build(), false); +} + +} // namespace + + +Y_UNIT_TEST_SUITE(TMiniKQLWideTakeSkipBlocks) { + Y_UNIT_TEST(TestWideTakeSkipBlocks) { + TSetup_ setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto flow = MakeFlow(setup); + + const auto part = pb.WideTakeBlocks(pb.WideSkipBlocks(flow, pb.NewDataLiteral<ui64>(3)), pb.NewDataLiteral<ui64>(5)); + const auto plain = pb.WideFromBlocks(part); + + const auto singleValueFlow = pb.NarrowMap(plain, [&](TRuntimeNode::TList items) -> TRuntimeNode { + // 0, 0; + // 1, 0; + // 2, 0; + // 3, 0; -> 3 + // 4, 0; -> 4 + // 5, 1; -> 6 + // 6, 1; -> 7 + // 7, 1; -> 8 + // 8, 1; + // 9, 1; + // 10, 1; + return pb.Add(items[0], items[1]); + }); + + const auto pgmReturn = pb.ForwardList(singleValueFlow); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 3); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 4); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 6); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 7); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 8); + } +} + +} // namespace NMiniKQL +} // namespace NKikimr + + diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_blocks_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_blocks_ut.cpp new file mode 100644 index 00000000000..da7a1dceed8 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_blocks_ut.cpp @@ -0,0 +1,737 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> + +#include <arrow/compute/exec_internal.h> +#include <arrow/array/builder_primitive.h> + +#include <ydb/library/yql/public/udf/udf_helpers.h> +#include <ydb/library/yql/public/udf/arrow/udf_arrow_helpers.h> + +BEGIN_SIMPLE_ARROW_UDF(TInc, i32(i32)) { + Y_UNUSED(valueBuilder); + const i32 arg = args[0].Get<i32>(); + return NYql::NUdf::TUnboxedValuePod(arg + 1); +} + +struct TIncKernelExec : public NYql::NUdf::TUnaryKernelExec<TIncKernelExec> { + template <typename TSink> + static void Process(NYql::NUdf::TBlockItem arg, const TSink& sink) { + sink(NYql::NUdf::TBlockItem(arg.As<i32>() + 1)); + } +}; + +END_SIMPLE_ARROW_UDF(TInc, TIncKernelExec::Do); + +SIMPLE_MODULE(TBlockUTModule, + TInc +) + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + arrow::Datum ExecuteOneKernel(const IArrowKernelComputationNode* kernelNode, + const std::vector<arrow::Datum>& argDatums, arrow::compute::ExecContext& execContext) { + const auto& kernel = kernelNode->GetArrowKernel(); + arrow::compute::KernelContext kernelContext(&execContext); + std::unique_ptr<arrow::compute::KernelState> state; + if (kernel.init) { + state = ARROW_RESULT(kernel.init(&kernelContext, { &kernel, kernelNode->GetArgsDesc(), nullptr })); + kernelContext.SetState(state.get()); + } + + auto executor = arrow::compute::detail::KernelExecutor::MakeScalar(); + ARROW_OK(executor->Init(&kernelContext, { &kernel, kernelNode->GetArgsDesc(), nullptr })); + auto listener = std::make_shared<arrow::compute::detail::DatumAccumulator>(); + ARROW_OK(executor->Execute(argDatums, listener.get())); + return executor->WrapResults(argDatums, listener->values()); + } + + void ExecuteAllKernels(std::vector<arrow::Datum>& datums, const TArrowKernelsTopology* topology, arrow::compute::ExecContext& execContext) { + for (ui32 i = 0; i < topology->Items.size(); ++i) { + std::vector<arrow::Datum> argDatums; + argDatums.reserve(topology->Items[i].Inputs.size()); + for (auto j : topology->Items[i].Inputs) { + argDatums.emplace_back(datums[j]); + } + + arrow::Datum output = ExecuteOneKernel(topology->Items[i].Node.get(), argDatums, execContext); + datums[i + topology->InputArgsCount] = output; + } + } +} + +Y_UNIT_TEST_SUITE(TMiniKQLBlocksTest) { +Y_UNIT_TEST(TestEmpty) { + TSetup<false> setup; + auto& pb = *setup.PgmBuilder; + + const auto type = pb.NewDataType(NUdf::TDataType<ui64>::Id); + const auto list = pb.NewEmptyList(type); + const auto sourceFlow = pb.ToFlow(list); + const auto flowAfterBlocks = pb.FromBlocks(pb.ToBlocks(sourceFlow)); + const auto pgmReturn = pb.ForwardList(flowAfterBlocks); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(!iterator.Next(item)); +} + +Y_UNIT_TEST(TestSimple) { + static const size_t dataCount = 1000; + TSetup<false> setup; + auto& pb = *setup.PgmBuilder; + + auto data = TVector<TRuntimeNode>(Reserve(dataCount)); + for (size_t i = 0; i < dataCount; ++i) { + data.push_back(pb.NewDataLiteral<ui64>(i)); + } + const auto type = pb.NewDataType(NUdf::TDataType<ui64>::Id); + const auto list = pb.NewList(type, data); + const auto sourceFlow = pb.ToFlow(list); + const auto flowAfterBlocks = pb.FromBlocks(pb.ToBlocks(sourceFlow)); + const auto pgmReturn = pb.ForwardList(flowAfterBlocks); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + + for (size_t i = 0; i < dataCount; ++i) { + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), i); + } + NUdf::TUnboxedValue item; + UNIT_ASSERT(!iterator.Next(item)); +} + +Y_UNIT_TEST(TestWideToBlocks) { + TSetup<false> setup; + auto& pb = *setup.PgmBuilder; + + const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id); + const auto tupleType = pb.NewTupleType({ui64Type, ui64Type}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(1), pb.NewDataLiteral<ui64>(10)}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(2), pb.NewDataLiteral<ui64>(20)}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(3), pb.NewDataLiteral<ui64>(30)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3}); + const auto flow = pb.ToFlow(list); + + const auto wideFlow = pb.ExpandMap(flow, [&](TRuntimeNode item) -> TRuntimeNode::TList { + return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; + }); + const auto wideBlocksFlow = pb.WideToBlocks(wideFlow); + const auto narrowBlocksFlow = pb.NarrowMap(wideBlocksFlow, [&](TRuntimeNode::TList items) -> TRuntimeNode { + return items[1]; + }); + const auto narrowFlow = pb.FromBlocks(narrowBlocksFlow); + const auto pgmReturn = pb.ForwardList(narrowFlow); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 10); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 20); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 30); +} + +namespace { +void TestChunked(bool withBlockExpand) { + TSetup<false> setup; + auto& pb = *setup.PgmBuilder; + + const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id); + const auto boolType = pb.NewDataType(NUdf::TDataType<bool>::Id); + const auto stringType = pb.NewDataType(NUdf::EDataSlot::String); + const auto utf8Type = pb.NewDataType(NUdf::EDataSlot::Utf8); + + const auto tupleType = pb.NewTupleType({ui64Type, boolType, stringType, utf8Type}); + + TVector<TRuntimeNode> items; + const size_t bigStrSize = 1024 * 1024 + 100; + const size_t smallStrSize = 256 * 1024; + for (size_t i = 0; i < 20; ++i) { + + if (i % 2 == 0) { + std::string big(bigStrSize, '0' + i); + std::string small(smallStrSize, 'A' + i); + + items.push_back(pb.NewTuple(tupleType, { pb.NewDataLiteral<ui64>(i), pb.NewDataLiteral<bool>(true), + pb.NewDataLiteral<NUdf::EDataSlot::String>(big), + pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(small), + })); + } else { + items.push_back(pb.NewTuple(tupleType, { pb.NewDataLiteral<ui64>(i), pb.NewDataLiteral<bool>(false), + pb.NewDataLiteral<NUdf::EDataSlot::String>(""), + pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(""), + })); + + } + } + + const auto list = pb.NewList(tupleType, std::move(items)); + + auto node = pb.ToFlow(list); + node = pb.ExpandMap(node, [&](TRuntimeNode item) -> TRuntimeNode::TList { + return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U), pb.Nth(item, 3U)}; + }); + node = pb.WideToBlocks(node); + if (withBlockExpand) { + node = pb.BlockExpandChunked(node); + // WideTakeBlocks won't work on chunked blocks + node = pb.WideTakeBlocks(node, pb.NewDataLiteral<ui64>(19)); + node = pb.WideFromBlocks(node); + } else { + // WideFromBlocks should support chunked blocks + node = pb.WideFromBlocks(node); + node = pb.Take(node, pb.NewDataLiteral<ui64>(19)); + } + node = pb.NarrowMap(node, [&](TRuntimeNode::TList items) -> TRuntimeNode { + return pb.NewTuple(tupleType, {items[0], items[1], items[2], items[3]}); + }); + + const auto pgmReturn = pb.ForwardList(node); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + + for (size_t i = 0; i < 19; ++i) { + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + ui64 num = item.GetElement(0).Get<ui64>(); + bool bl = item.GetElement(1).Get<bool>(); + auto strVal = item.GetElement(2); + auto utf8Val = item.GetElement(3); + std::string_view str = strVal.AsStringRef(); + std::string_view utf8 = utf8Val.AsStringRef(); + + UNIT_ASSERT_VALUES_EQUAL(num, i); + UNIT_ASSERT_VALUES_EQUAL(bl, i % 2 == 0); + if (i % 2 == 0) { + std::string big(bigStrSize, '0' + i); + std::string small(smallStrSize, 'A' + i); + UNIT_ASSERT_VALUES_EQUAL(str, big); + UNIT_ASSERT_VALUES_EQUAL(utf8, small); + } else { + UNIT_ASSERT_VALUES_EQUAL(str.size(), 0); + UNIT_ASSERT_VALUES_EQUAL(utf8.size(), 0); + } + } + + NUdf::TUnboxedValue item; + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + +} + +} // namespace + +Y_UNIT_TEST(TestBlockExpandChunked) { + TestChunked(true); +} + +Y_UNIT_TEST(TestWideFromBlocksForChunked) { + TestChunked(false); +} + +Y_UNIT_TEST(TestScalar) { + const ui64 testValue = 42; + + TSetup<false> setup; + auto& pb = *setup.PgmBuilder; + + auto dataLiteral = pb.NewDataLiteral<ui64>(testValue); + const auto dataAfterBlocks = pb.AsScalar(dataLiteral); + + const auto graph = setup.BuildGraph(dataAfterBlocks); + const auto value = graph->GetValue(); + UNIT_ASSERT(value.HasValue() && value.IsBoxed()); + UNIT_ASSERT_VALUES_EQUAL(TArrowBlock::From(value).GetDatum().scalar_as<arrow::UInt64Scalar>().value, testValue); +} + +Y_UNIT_TEST(TestBlockFunc) { + TSetup<false> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id); + const auto tupleType = pb.NewTupleType({ui64Type, ui64Type}); + const auto ui64BlockType = pb.NewBlockType(ui64Type, TBlockType::EShape::Many); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(1), pb.NewDataLiteral<ui64>(10)}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(2), pb.NewDataLiteral<ui64>(20)}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(3), pb.NewDataLiteral<ui64>(30)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3}); + const auto flow = pb.ToFlow(list); + + const auto wideFlow = pb.ExpandMap(flow, [&](TRuntimeNode item) -> TRuntimeNode::TList { + return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; + }); + const auto wideBlocksFlow = pb.WideToBlocks(wideFlow); + const auto sumWideFlow = pb.WideMap(wideBlocksFlow, [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { + return {pb.BlockFunc("Add", ui64BlockType, {items[0], items[1]})}; + }); + const auto sumNarrowFlow = pb.NarrowMap(sumWideFlow, [&](TRuntimeNode::TList items) -> TRuntimeNode { + return items[0]; + }); + const auto pgmReturn = pb.Collect(pb.FromBlocks(sumNarrowFlow)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 11); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 22); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 33); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); +} + +Y_UNIT_TEST(TestBlockFuncWithNullables) { + TSetup<false> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto optionalUi64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id, true); + const auto tupleType = pb.NewTupleType({optionalUi64Type, optionalUi64Type}); + const auto emptyOptionalUi64 = pb.NewEmptyOptional(optionalUi64Type); + const auto ui64OptBlockType = pb.NewBlockType(optionalUi64Type, TBlockType::EShape::Many); + + const auto data1 = pb.NewTuple(tupleType, { + pb.NewOptional(pb.NewDataLiteral<ui64>(1)), + emptyOptionalUi64 + }); + const auto data2 = pb.NewTuple(tupleType, { + emptyOptionalUi64, + pb.NewOptional(pb.NewDataLiteral<ui64>(20)) + }); + const auto data3 = pb.NewTuple(tupleType, { + emptyOptionalUi64, + emptyOptionalUi64 + }); + const auto data4 = pb.NewTuple(tupleType, { + pb.NewOptional(pb.NewDataLiteral<ui64>(10)), + pb.NewOptional(pb.NewDataLiteral<ui64>(20)) + }); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + const auto flow = pb.ToFlow(list); + + const auto wideFlow = pb.ExpandMap(flow, [&](TRuntimeNode item) -> TRuntimeNode::TList { + return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; + }); + const auto wideBlocksFlow = pb.WideToBlocks(wideFlow); + const auto sumWideFlow = pb.WideMap(wideBlocksFlow, [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { + return {pb.BlockFunc("Add", ui64OptBlockType, {items[0], items[1]})}; + }); + const auto sumNarrowFlow = pb.NarrowMap(sumWideFlow, [&](TRuntimeNode::TList items) -> TRuntimeNode { + return items[0]; + }); + const auto pgmReturn = pb.Collect(pb.FromBlocks(sumNarrowFlow)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 30); + UNIT_ASSERT(!iterator.Next(item)); +} + +Y_UNIT_TEST(TestBlockFuncWithNullableScalar) { + TSetup<false> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto optionalUi64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id, true); + const auto ui64OptBlockType = pb.NewBlockType(optionalUi64Type, TBlockType::EShape::Many); + const auto emptyOptionalUi64 = pb.NewEmptyOptional(optionalUi64Type); + + const auto list = pb.NewList(optionalUi64Type, { + pb.NewOptional(pb.NewDataLiteral<ui64>(10)), + pb.NewOptional(pb.NewDataLiteral<ui64>(20)), + pb.NewOptional(pb.NewDataLiteral<ui64>(30)) + }); + const auto flow = pb.ToFlow(list); + const auto blocksFlow = pb.ToBlocks(flow); + + THolder<IComputationGraph> graph; + auto map = [&](const TProgramBuilder::TUnaryLambda& func) { + const auto pgmReturn = pb.Collect(pb.FromBlocks(pb.Map(blocksFlow, func))); + graph = setup.BuildGraph(pgmReturn); + return graph->GetValue().GetListIterator(); + }; + + { + const auto scalar = pb.AsScalar(emptyOptionalUi64); + auto iterator = map([&](TRuntimeNode item) -> TRuntimeNode { + return {pb.BlockFunc("Add", ui64OptBlockType, {scalar, item})}; + }); + + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + UNIT_ASSERT(!iterator.Next(item)); + } + + { + const auto scalar = pb.AsScalar(emptyOptionalUi64); + auto iterator = map([&](TRuntimeNode item) -> TRuntimeNode { + return {pb.BlockFunc("Add", ui64OptBlockType, {item, scalar})}; + }); + + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + UNIT_ASSERT(!iterator.Next(item)); + } + + { + const auto scalar = pb.AsScalar(pb.NewDataLiteral<ui64>(100)); + auto iterator = map([&](TRuntimeNode item) -> TRuntimeNode { + return {pb.BlockFunc("Add", ui64OptBlockType, {item, scalar})}; + }); + + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 110); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 120); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 130); + + UNIT_ASSERT(!iterator.Next(item)); + } +} + +Y_UNIT_TEST(TestBlockFuncWithScalar) { + TSetup<false> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id); + const auto ui64BlockType = pb.NewBlockType(ui64Type, TBlockType::EShape::Many); + + const auto data1 = pb.NewDataLiteral<ui64>(10); + const auto data2 = pb.NewDataLiteral<ui64>(20); + const auto data3 = pb.NewDataLiteral<ui64>(30); + const auto rightScalar = pb.AsScalar(pb.NewDataLiteral<ui64>(100)); + const auto leftScalar = pb.AsScalar(pb.NewDataLiteral<ui64>(1000)); + + const auto list = pb.NewList(ui64Type, {data1, data2, data3}); + const auto flow = pb.ToFlow(list); + const auto blocksFlow = pb.ToBlocks(flow); + const auto sumBlocksFlow = pb.Map(blocksFlow, [&](TRuntimeNode item) -> TRuntimeNode { + return {pb.BlockFunc("Add", ui64BlockType, { leftScalar, {pb.BlockFunc("Add", ui64BlockType, { item, rightScalar } )}})}; + }); + const auto pgmReturn = pb.Collect(pb.FromBlocks(sumBlocksFlow)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 1110); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 1120); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 1130); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); +} + +Y_UNIT_TEST(TestWideFromBlocks) { + TSetup<false> setup; + auto& pb = *setup.PgmBuilder; + + const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id); + const auto tupleType = pb.NewTupleType({ui64Type, ui64Type}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(1), pb.NewDataLiteral<ui64>(10)}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(2), pb.NewDataLiteral<ui64>(20)}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(3), pb.NewDataLiteral<ui64>(30)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3}); + const auto flow = pb.ToFlow(list); + + const auto wideFlow = pb.ExpandMap(flow, [&](TRuntimeNode item) -> TRuntimeNode::TList { + return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; + }); + const auto wideBlocksFlow = pb.WideToBlocks(wideFlow); + const auto wideFlow2 = pb.WideFromBlocks(wideBlocksFlow); + const auto narrowFlow = pb.NarrowMap(wideFlow2, [&](TRuntimeNode::TList items) -> TRuntimeNode { + return items[1]; + }); + + const auto pgmReturn = pb.ForwardList(narrowFlow); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 10); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 20); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 30); +} +} + +Y_UNIT_TEST_SUITE(TMiniKQLDirectKernelTest) { +Y_UNIT_TEST(Simple) { + TSetup<false> setup; + auto& pb = *setup.PgmBuilder; + + const auto boolType = pb.NewDataType(NUdf::TDataType<bool>::Id); + const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id); + const auto boolBlocksType = pb.NewBlockType(boolType, TBlockType::EShape::Many); + const auto ui64BlocksType = pb.NewBlockType(ui64Type, TBlockType::EShape::Many); + const auto arg1 = pb.Arg(boolBlocksType); + const auto arg2 = pb.Arg(ui64BlocksType); + const auto arg3 = pb.Arg(ui64BlocksType); + const auto ifNode = pb.BlockIf(arg1, arg2, arg3); + const auto eqNode = pb.BlockFunc("Equals", boolBlocksType, { ifNode, arg2 }); + + const auto graph = setup.BuildGraph(eqNode, {arg1.GetNode(), arg2.GetNode(), arg3.GetNode()}); + const auto topology = graph->GetKernelsTopology(); + UNIT_ASSERT(topology); + UNIT_ASSERT_VALUES_EQUAL(topology->InputArgsCount, 3); + UNIT_ASSERT_VALUES_EQUAL(topology->Items.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(topology->Items[0].Node->GetKernelName(), "If"); + const std::vector<ui32> expectedInputs1{{0, 1, 2}}; + UNIT_ASSERT_VALUES_EQUAL(topology->Items[0].Inputs, expectedInputs1); + UNIT_ASSERT_VALUES_EQUAL(topology->Items[1].Node->GetKernelName(), "Equals"); + const std::vector<ui32> expectedInputs2{{3, 1}}; + UNIT_ASSERT_VALUES_EQUAL(topology->Items[1].Inputs, expectedInputs2); + + arrow::compute::ExecContext execContext; + const size_t blockSize = 100000; + std::vector<arrow::Datum> datums(topology->InputArgsCount + topology->Items.size()); + { + arrow::UInt8Builder builder1(execContext.memory_pool()); + arrow::UInt64Builder builder2(execContext.memory_pool()), builder3(execContext.memory_pool()); + ARROW_OK(builder1.Reserve(blockSize)); + ARROW_OK(builder2.Reserve(blockSize)); + ARROW_OK(builder3.Reserve(blockSize)); + for (size_t i = 0; i < blockSize; ++i) { + builder1.UnsafeAppend(i & 1); + builder2.UnsafeAppend(i); + builder3.UnsafeAppend(3 * i); + } + + std::shared_ptr<arrow::ArrayData> data1; + ARROW_OK(builder1.FinishInternal(&data1)); + std::shared_ptr<arrow::ArrayData> data2; + ARROW_OK(builder2.FinishInternal(&data2)); + std::shared_ptr<arrow::ArrayData> data3; + ARROW_OK(builder3.FinishInternal(&data3)); + datums[0] = data1; + datums[1] = data2; + datums[2] = data3; + } + + ExecuteAllKernels(datums, topology, execContext); + + auto res = datums.back().array()->GetValues<ui8>(1); + for (size_t i = 0; i < blockSize; ++i) { + auto expected = (((i & 1) ? i : i * 3) == i) ? 1 : 0; + UNIT_ASSERT_VALUES_EQUAL(res[i], expected); + } +} + +Y_UNIT_TEST(WithScalars) { + TSetup<false> setup; + auto& pb = *setup.PgmBuilder; + + const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id); + const auto ui64BlocksType = pb.NewBlockType(ui64Type, TBlockType::EShape::Many); + const auto scalar = pb.AsScalar(pb.NewDataLiteral(false)); + const auto arg1 = pb.Arg(ui64BlocksType); + const auto arg2 = pb.Arg(ui64BlocksType); + const auto ifNode = pb.BlockIf(scalar, arg1, arg2); + + const auto graph = setup.BuildGraph(ifNode, {arg1.GetNode(), arg2.GetNode()}); + const auto topology = graph->GetKernelsTopology(); + UNIT_ASSERT(topology); + UNIT_ASSERT_VALUES_EQUAL(topology->InputArgsCount, 2); + UNIT_ASSERT_VALUES_EQUAL(topology->Items.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(topology->Items[0].Node->GetKernelName(), "AsScalar"); + const std::vector<ui32> expectedInputs1; + UNIT_ASSERT_VALUES_EQUAL(topology->Items[0].Inputs, expectedInputs1); + UNIT_ASSERT_VALUES_EQUAL(topology->Items[1].Node->GetKernelName(), "If"); + const std::vector<ui32> expectedInputs2{{2, 0, 1}}; + UNIT_ASSERT_VALUES_EQUAL(topology->Items[1].Inputs, expectedInputs2); + + arrow::compute::ExecContext execContext; + const size_t blockSize = 100000; + std::vector<arrow::Datum> datums(topology->InputArgsCount + topology->Items.size()); + { + arrow::UInt64Builder builder1(execContext.memory_pool()), builder2(execContext.memory_pool()); + ARROW_OK(builder1.Reserve(blockSize)); + ARROW_OK(builder2.Reserve(blockSize)); + for (size_t i = 0; i < blockSize; ++i) { + builder1.UnsafeAppend(i); + builder2.UnsafeAppend(3 * i); + } + + std::shared_ptr<arrow::ArrayData> data1; + ARROW_OK(builder1.FinishInternal(&data1)); + std::shared_ptr<arrow::ArrayData> data2; + ARROW_OK(builder2.FinishInternal(&data2)); + datums[0] = data1; + datums[1] = data2; + } + + ExecuteAllKernels(datums, topology, execContext); + + auto res = datums.back().array()->GetValues<ui64>(1); + for (size_t i = 0; i < blockSize; ++i) { + auto expected = 3 * i; + UNIT_ASSERT_VALUES_EQUAL(res[i], expected); + } +} + +Y_UNIT_TEST(Udf) { + TVector<TUdfModuleInfo> modules; + modules.emplace_back(TUdfModuleInfo{"", "BlockUT", new TBlockUTModule()}); + TSetup<false> setup({}, std::move(modules)); + + auto& pb = *setup.PgmBuilder; + + const auto i32Type = pb.NewDataType(NUdf::TDataType<i32>::Id); + const auto i32BlocksType = pb.NewBlockType(i32Type, TBlockType::EShape::Many); + const auto arg1 = pb.Arg(i32BlocksType); + const auto userType = pb.NewTupleType({ + pb.NewTupleType({i32BlocksType}), + pb.NewEmptyStructType(), + pb.NewEmptyTupleType()}); + const auto udf = pb.Udf("BlockUT.Inc_BlocksImpl", pb.NewVoid(), userType); + const auto apply = pb.Apply(udf, {arg1}); + + const auto graph = setup.BuildGraph(apply, {arg1.GetNode() }); + const auto topology = graph->GetKernelsTopology(); + UNIT_ASSERT(topology); + UNIT_ASSERT_VALUES_EQUAL(topology->InputArgsCount, 1); + UNIT_ASSERT_VALUES_EQUAL(topology->Items.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(topology->Items[0].Node->GetKernelName(), "Apply"); + const std::vector<ui32> expectedInputs1{0}; + UNIT_ASSERT_VALUES_EQUAL(topology->Items[0].Inputs, expectedInputs1); + + arrow::compute::ExecContext execContext; + const size_t blockSize = 100000; + std::vector<arrow::Datum> datums(topology->InputArgsCount + topology->Items.size()); + { + arrow::Int32Builder builder1(execContext.memory_pool()); + ARROW_OK(builder1.Reserve(blockSize)); + for (size_t i = 0; i < blockSize; ++i) { + builder1.UnsafeAppend(i); + } + + std::shared_ptr<arrow::ArrayData> data1; + ARROW_OK(builder1.FinishInternal(&data1)); + datums[0] = data1; + } + + ExecuteAllKernels(datums, topology, execContext); + + auto res = datums.back().array()->GetValues<i32>(1); + for (size_t i = 0; i < blockSize; ++i) { + auto expected = i + 1; + UNIT_ASSERT_VALUES_EQUAL(res[i], expected); + } +} + +Y_UNIT_TEST(ScalarApply) { + TSetup<false> setup; + auto& pb = *setup.PgmBuilder; + + const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id); + const auto ui64BlocksType = pb.NewBlockType(ui64Type, TBlockType::EShape::Many); + const auto arg1 = pb.Arg(ui64BlocksType); + const auto arg2 = pb.Arg(ui64BlocksType); + const auto scalarApply = pb.ScalarApply({arg1,arg2}, [&](auto args){ + return pb.Add(args[0], args[1]); + }); + + const auto graph = setup.BuildGraph(scalarApply, {arg1.GetNode(), arg2.GetNode()}); + const auto topology = graph->GetKernelsTopology(); + UNIT_ASSERT(topology); + UNIT_ASSERT_VALUES_EQUAL(topology->InputArgsCount, 2); + UNIT_ASSERT_VALUES_EQUAL(topology->Items.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(topology->Items[0].Node->GetKernelName(), "ScalarApply"); + const std::vector<ui32> expectedInputs1{{0, 1}}; + UNIT_ASSERT_VALUES_EQUAL(topology->Items[0].Inputs, expectedInputs1); + + arrow::compute::ExecContext execContext; + const size_t blockSize = 100000; + std::vector<arrow::Datum> datums(topology->InputArgsCount + topology->Items.size()); + { + arrow::UInt64Builder builder1(execContext.memory_pool()), builder2(execContext.memory_pool()); + ARROW_OK(builder1.Reserve(blockSize)); + ARROW_OK(builder2.Reserve(blockSize)); + for (size_t i = 0; i < blockSize; ++i) { + builder1.UnsafeAppend(i); + builder2.UnsafeAppend(2 * i); + } + + std::shared_ptr<arrow::ArrayData> data1; + ARROW_OK(builder1.FinishInternal(&data1)); + std::shared_ptr<arrow::ArrayData> data2; + ARROW_OK(builder2.FinishInternal(&data2)); + datums[0] = data1; + datums[1] = data2; + } + + ExecuteAllKernels(datums, topology, execContext); + + auto res = datums.back().array()->GetValues<ui64>(1); + for (size_t i = 0; i < blockSize; ++i) { + auto expected = 3 * i; + UNIT_ASSERT_VALUES_EQUAL(res[i], expected); + } +} + +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_chain_map_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_chain_map_ut.cpp new file mode 100644 index 00000000000..b41a0527de8 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_chain_map_ut.cpp @@ -0,0 +1,254 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> + +namespace NKikimr { +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLChainMapNodeTest) { + Y_UNIT_TEST_LLVM(TestOverList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto dataType = pb.NewOptionalType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id)})); + + auto data0 = pb.NewEmptyOptional(dataType); + + auto data2 = pb.NewOptional(pb.NewTuple({ + pb.NewDataLiteral<i32>(7), + pb.NewDataLiteral<NUdf::EDataSlot::String>("A") + })); + auto data3 = pb.NewOptional(pb.NewTuple({ + pb.NewDataLiteral<i32>(1), + pb.NewDataLiteral<NUdf::EDataSlot::String>("D") + })); + + auto list = pb.NewList(dataType, {data2, data0, data3}); + + auto init = pb.NewTuple({ + pb.NewOptional(pb.NewDataLiteral<i32>(3)), + pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("B")) + }); + + auto pgmReturn = pb.ChainMap(list, init, + [&](TRuntimeNode item, TRuntimeNode state) -> TRuntimeNodePair { + auto key = pb.Nth(item, 0); + auto val = pb.Nth(item, 1); + auto skey = pb.AggrAdd(pb.Nth(state, 0), key); + auto sval = pb.AggrConcat(pb.Nth(state, 1), val); + return {pb.NewTuple({key, val, skey, sval}), pb.NewTuple({skey, sval})}; + } + ); + + auto graph = setup.BuildGraph(pgmReturn); + auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 7); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "A"); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 10); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BA"); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 10); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BA"); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "D"); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 11); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BAD"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(Test1OverList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto dataType = pb.NewOptionalType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id)})); + + auto data0 = pb.NewEmptyOptional(dataType); + + auto data1 = pb.NewOptional(pb.NewTuple({ + pb.NewDataLiteral<i32>(3), + pb.NewDataLiteral<NUdf::EDataSlot::String>("B") + })); + auto data2 = pb.NewOptional(pb.NewTuple({ + pb.NewDataLiteral<i32>(7), + pb.NewDataLiteral<NUdf::EDataSlot::String>("A") + })); + auto data3 = pb.NewOptional(pb.NewTuple({ + pb.NewDataLiteral<i32>(1), + pb.NewDataLiteral<NUdf::EDataSlot::String>("D") + })); + + auto list = pb.NewList(dataType, {data1, data2, data3, data0}); + + auto pgmReturn = pb.Chain1Map(list, + [&](TRuntimeNode item) -> TRuntimeNodePair { + auto key = pb.Nth(item, 0); + auto val = pb.Nth(item, 1); + return {pb.NewTuple({key, val, key, val}), pb.NewTuple({key, val})}; + }, + [&](TRuntimeNode item, TRuntimeNode state) -> TRuntimeNodePair { + auto key = pb.Nth(item, 0); + auto val = pb.Nth(item, 1); + auto skey = pb.Add(pb.Nth(state, 0), key); + auto sval = pb.Concat(pb.Nth(state, 1), val); + return {pb.NewTuple({key, val, skey, sval}), pb.NewTuple({skey, sval})}; + } + ); + + auto graph = setup.BuildGraph(pgmReturn); + auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 3); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "B"); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 3); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "B"); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 7); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "A"); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 10); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BA"); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "D"); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 11); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BAD"); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(!item.GetElement(2)); + UNIT_ASSERT(!item.GetElement(3)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestOverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto dataType = pb.NewOptionalType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id)})); + + auto data0 = pb.NewEmptyOptional(dataType); + + auto data2 = pb.NewOptional(pb.NewTuple({ + pb.NewDataLiteral<i32>(7), + pb.NewDataLiteral<NUdf::EDataSlot::String>("A") + })); + auto data3 = pb.NewOptional(pb.NewTuple({ + pb.NewDataLiteral<i32>(1), + pb.NewDataLiteral<NUdf::EDataSlot::String>("D") + })); + + auto list = pb.NewList(dataType, {data2, data0, data3}); + + auto init = pb.NewTuple({ + pb.NewOptional(pb.NewDataLiteral<i32>(3)), + pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("B")) + }); + + auto pgmReturn = pb.FromFlow(pb.ChainMap(pb.ToFlow(list), init, + [&](TRuntimeNode item, TRuntimeNode state) -> TRuntimeNodePair { + auto key = pb.Nth(item, 0); + auto val = pb.Nth(item, 1); + auto skey = pb.AggrAdd(pb.Nth(state, 0), key); + auto sval = pb.AggrConcat(pb.Nth(state, 1), val); + return {pb.NewTuple({key, val, skey, sval}), pb.NewTuple({skey, sval})}; + } + )); + + auto graph = setup.BuildGraph(pgmReturn); + auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 7); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "A"); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 10); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BA"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 10); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BA"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "D"); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 11); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BAD"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(Test1OverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto dataType = pb.NewOptionalType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id)})); + + auto data0 = pb.NewEmptyOptional(dataType); + + auto data1 = pb.NewOptional(pb.NewTuple({ + pb.NewDataLiteral<i32>(3), + pb.NewDataLiteral<NUdf::EDataSlot::String>("B") + })); + auto data2 = pb.NewOptional(pb.NewTuple({ + pb.NewDataLiteral<i32>(7), + pb.NewDataLiteral<NUdf::EDataSlot::String>("A") + })); + auto data3 = pb.NewOptional(pb.NewTuple({ + pb.NewDataLiteral<i32>(1), + pb.NewDataLiteral<NUdf::EDataSlot::String>("D") + })); + + auto list = pb.NewList(dataType, {data1, data2, data3, data0}); + + auto pgmReturn = pb.FromFlow(pb.Chain1Map(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNodePair { + auto key = pb.Nth(item, 0); + auto val = pb.Nth(item, 1); + return {pb.NewTuple({key, val, key, val}), pb.NewTuple({key, val})}; + }, + [&](TRuntimeNode item, TRuntimeNode state) -> TRuntimeNodePair { + auto key = pb.Nth(item, 0); + auto val = pb.Nth(item, 1); + auto skey = pb.Add(pb.Nth(state, 0), key); + auto sval = pb.Concat(pb.Nth(state, 1), val); + return {pb.NewTuple({key, val, skey, sval}), pb.NewTuple({skey, sval})}; + } + )); + + auto graph = setup.BuildGraph(pgmReturn); + auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 3); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "B"); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 3); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "B"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 7); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "A"); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 10); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BA"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "D"); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 11); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BAD"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(!item.GetElement(2)); + UNIT_ASSERT(!item.GetElement(3)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp new file mode 100644 index 00000000000..8d9d5576696 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp @@ -0,0 +1,487 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + +template<bool UseLLVM> +TRuntimeNode MakeStream(TSetup<UseLLVM>& setup, ui64 count = 9U) { + TProgramBuilder& pb = *setup.PgmBuilder; + + TCallableBuilder callableBuilder(*setup.Env, "TestStream", + pb.NewStreamType( + pb.NewDataType(NUdf::EDataSlot::Uint64) + ) + ); + + callableBuilder.Add(pb.NewDataLiteral(count)); + + return TRuntimeNode(callableBuilder.Build(), false); +} + +template<bool UseLLVM> +TRuntimeNode MakeFlow(TSetup<UseLLVM>& setup, ui64 count = 9U) { + TProgramBuilder& pb = *setup.PgmBuilder; + return pb.ToFlow(MakeStream<UseLLVM>(setup, count)); +} + +template<bool UseLLVM> +TRuntimeNode GroupWithBomb(TSetup<UseLLVM>& setup, TRuntimeNode stream) { + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto keyExtractor = [&](TRuntimeNode item) { return item; }; + const auto groupSwitch = [&](TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral<bool>(false); }; + + return pb.Chopper(stream, keyExtractor, groupSwitch, [&](TRuntimeNode, TRuntimeNode group) { + const auto bomb = pb.NewDataLiteral<NUdf::EDataSlot::String>("BOMB"); + return pb.Ensure(pb.Map(group, [&] (TRuntimeNode) { return bomb; }), pb.NewDataLiteral<bool>(false), bomb, "", 0, 0); + }); +} + +template<bool UseLLVM> +TRuntimeNode Group(TSetup<UseLLVM>& setup, TRuntimeNode stream, const std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)>& groupSwitch) { + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto keyExtractor = [&](TRuntimeNode item) { return item; }; + + return pb.Chopper(stream, keyExtractor, groupSwitch, [&](TRuntimeNode key, TRuntimeNode grpItem) { + return pb.Condense(grpItem, pb.NewDataLiteral<NUdf::EDataSlot::String>("*"), + [&] (TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral<bool>(false); }, + [&] (TRuntimeNode item, TRuntimeNode state) { + auto res = pb.Concat(pb.ToString(key), pb.ToString(item)); + res = pb.Concat(state, res); + return pb.Concat(res, pb.NewDataLiteral<NUdf::EDataSlot::String>("*")); + }); + }); +} + +template<bool UseLLVM> +TRuntimeNode GroupGetKeysFirst(TSetup<UseLLVM>& setup, TRuntimeNode stream, const std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)>& groupSwitch) { + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto keyExtractor = [&](TRuntimeNode item) { return item; }; + + const bool isFlow = stream.GetStaticType()->IsFlow(); + + return pb.Chopper(stream, keyExtractor, groupSwitch, [&](TRuntimeNode key, TRuntimeNode grpItem) { + auto list = pb.ToList(pb.AggrConcat( + pb.NewOptional(pb.Concat(pb.ToString(key), pb.NewDataLiteral<NUdf::EDataSlot::String>(":"))), + pb.ToOptional(pb.Collect(pb.Condense1(grpItem, + [&] (TRuntimeNode item) { return pb.ToString(item); }, + [&] (TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral<bool>(false); }, + [&] (TRuntimeNode item, TRuntimeNode state) { + return pb.Concat(state, pb.ToString(item)); + } + ))) + )); + return isFlow ? pb.ToFlow(list) : pb.Iterator(list, {}); + }); +} + +template<bool UseLLVM> +TRuntimeNode GroupKeys(TSetup<UseLLVM>& setup, TRuntimeNode stream, const std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)>& groupSwitch) { + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto keyExtractor = [&](TRuntimeNode item) { return item; }; + + return pb.Chopper(stream, keyExtractor, groupSwitch, + [&](TRuntimeNode key, TRuntimeNode group) { + return pb.Map(pb.Take(group, pb.NewDataLiteral<ui64>(1ULL)), [&](TRuntimeNode) { return pb.ToString(key); }); + } + ); +} + +template<bool UseLLVM> +TRuntimeNode StreamToString(TSetup<UseLLVM>& setup, TRuntimeNode stream) { + TProgramBuilder& pb = *setup.PgmBuilder; + + stream = pb.Condense(stream, pb.NewDataLiteral<NUdf::EDataSlot::String>("|"), + [&] (TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral(false); }, + [&] (TRuntimeNode item, TRuntimeNode state) { + return pb.Concat(pb.Concat(state, item), pb.NewDataLiteral<NUdf::EDataSlot::String>("|")); + } + ); + if (stream.GetStaticType()->IsFlow()) { + stream = pb.FromFlow(stream); + } + return stream; +} + +} // unnamed + +Y_UNIT_TEST_SUITE(TMiniKQLChopperStreamTest) { + Y_UNIT_TEST_LLVM(TestEmpty) { + TSetup<LLVM> setup; + + const auto stream = GroupWithBomb(setup, MakeStream(setup, 0U)); + const auto pgm = StreamToString(setup, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|"); + } + + Y_UNIT_TEST_LLVM(TestGrouping) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto stream = MakeStream(setup); + stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) { + Y_UNUSED(key); + return pb.Equals(item, pb.NewDataLiteral<ui64>(0)); + }); + const auto pgm = StreamToString(setup, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|"); + } + + Y_UNIT_TEST_LLVM(TestGroupingGetKeysFirst) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto stream = MakeStream(setup); + stream = GroupGetKeysFirst(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) { + Y_UNUSED(key); + return pb.Equals(item, pb.NewDataLiteral<ui64>(0)); + }); + const auto pgm = StreamToString(setup, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0:0|0:01|0:0|0:0|0:0123|"); + } + + Y_UNIT_TEST_LLVM(TestGroupingKeyNotEquals) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto stream = MakeStream(setup); + stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) { + return pb.NotEquals(item, key); + }); + const auto pgm = StreamToString(setup, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*00*|*11*|*00*00*00*|*11*|*22*|*33*|"); + } + + Y_UNIT_TEST_LLVM(TestGroupingWithEmptyInput) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto stream = MakeStream(setup, 0); + stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) { + Y_UNUSED(key); + return pb.Equals(item, pb.NewDataLiteral<ui64>(0)); + }); + const auto pgm = StreamToString(setup, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|"); + } + + Y_UNIT_TEST_LLVM(TestSingleGroup) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto stream = MakeStream(setup); + stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) { + Y_UNUSED(key); + Y_UNUSED(item); + return pb.NewDataLiteral<bool>(false); + }); + const auto pgm = StreamToString(setup, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*00*01*00*00*00*01*02*03*|"); + } + + Y_UNIT_TEST_LLVM(TestGroupingWithYield) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto stream = MakeStream(setup); + TSwitchInput switchInput; + switchInput.Indicies.push_back(0); + switchInput.InputType = stream.GetStaticType(); + + stream = pb.Switch(stream, + MakeArrayRef(&switchInput, 1), + [&](ui32 /*index*/, TRuntimeNode item1) { + return Group(setup, item1, [&](TRuntimeNode key, TRuntimeNode item2) { + Y_UNUSED(key); + return pb.Equals(item2, pb.NewDataLiteral<ui64>(0)); + }); + }, + 1, + pb.NewStreamType(pb.NewDataType(NUdf::EDataSlot::String)) + ); + + const auto pgm = StreamToString(setup, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|"); + } + + Y_UNIT_TEST_LLVM(TestGroupingWithCutSubStreams) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto stream = MakeStream(setup); + + stream = GroupKeys(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) { + Y_UNUSED(key); + return pb.Equals(item, pb.NewDataLiteral<ui64>(0)); + }); + + const auto pgm = StreamToString(setup, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|"); + } + + Y_UNIT_TEST_LLVM(TestGroupingWithYieldAndCutSubStreams) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto stream = MakeStream(setup); + TSwitchInput switchInput; + switchInput.Indicies.push_back(0); + switchInput.InputType = stream.GetStaticType(); + + stream = pb.Switch(stream, + MakeArrayRef(&switchInput, 1), + [&](ui32 /*index*/, TRuntimeNode item1) { + return GroupKeys(setup, item1, [&](TRuntimeNode key, TRuntimeNode item2) { + Y_UNUSED(key); + return pb.Equals(item2, pb.NewDataLiteral<ui64>(0)); + }); + }, + 1, + pb.NewStreamType(pb.NewDataType(NUdf::EDataSlot::String)) + ); + + const auto pgm = StreamToString(setup, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|"); + } +} +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 9u +Y_UNIT_TEST_SUITE(TMiniKQLChopperFlowTest) { + Y_UNIT_TEST_LLVM(TestEmpty) { + TSetup<LLVM> setup; + + const auto stream = GroupWithBomb(setup, MakeFlow(setup, 0U)); + const auto pgm = StreamToString(setup, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|"); + } + + Y_UNIT_TEST_LLVM(TestGrouping) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto stream = MakeFlow(setup); + stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) { + Y_UNUSED(key); + return pb.Equals(item, pb.NewDataLiteral<ui64>(0)); + }); + const auto pgm = StreamToString(setup, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|"); + } + + Y_UNIT_TEST_LLVM(TestGroupingGetKeysFirst) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto stream = MakeFlow(setup); + stream = GroupGetKeysFirst(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) { + Y_UNUSED(key); + return pb.Equals(item, pb.NewDataLiteral<ui64>(0)); + }); + const auto pgm = StreamToString(setup, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0:0|0:01|0:0|0:0|0:0123|"); + } + + Y_UNIT_TEST_LLVM(TestGroupingKeyNotEquals) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto stream = MakeFlow(setup); + stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) { + return pb.NotEquals(item, key); + }); + const auto pgm = StreamToString(setup, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*00*|*11*|*00*00*00*|*11*|*22*|*33*|"); + } + + Y_UNIT_TEST_LLVM(TestGroupingWithEmptyInput) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto stream = MakeFlow(setup, 0); + stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) { + Y_UNUSED(key); + return pb.Equals(item, pb.NewDataLiteral<ui64>(0)); + }); + const auto pgm = StreamToString(setup, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|"); + } + + Y_UNIT_TEST_LLVM(TestSingleGroup) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto stream = MakeFlow(setup); + stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) { + Y_UNUSED(key); + Y_UNUSED(item); + return pb.NewDataLiteral<bool>(false); + }); + const auto pgm = StreamToString(setup, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*00*01*00*00*00*01*02*03*|"); + } + + Y_UNIT_TEST_LLVM(TestGroupingWithYield) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto stream = MakeFlow(setup); + TSwitchInput switchInput; + switchInput.Indicies.push_back(0); + switchInput.InputType = stream.GetStaticType(); + + stream = pb.Switch(stream, + MakeArrayRef(&switchInput, 1), + [&](ui32 /*index*/, TRuntimeNode item1) { + return Group(setup, item1, [&](TRuntimeNode key, TRuntimeNode item2) { + Y_UNUSED(key); + return pb.Equals(item2, pb.NewDataLiteral<ui64>(0)); + }); + }, + 1, + pb.NewFlowType(pb.NewDataType(NUdf::EDataSlot::String)) + ); + + const auto pgm = StreamToString(setup, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|"); + } + + Y_UNIT_TEST_LLVM(TestGroupingWithCutSubStreams) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto stream = MakeFlow(setup); + + stream = GroupKeys(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) { + Y_UNUSED(key); + return pb.Equals(item, pb.NewDataLiteral<ui64>(0)); + }); + + const auto pgm = StreamToString(setup, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|"); + } + + Y_UNIT_TEST_LLVM(TestGroupingWithYieldAndCutSubStreams) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto stream = MakeFlow(setup); + TSwitchInput switchInput; + switchInput.Indicies.push_back(0); + switchInput.InputType = stream.GetStaticType(); + + stream = pb.Switch(stream, + MakeArrayRef(&switchInput, 1), + [&](ui32 /*index*/, TRuntimeNode item1) { + return GroupKeys(setup, item1, [&](TRuntimeNode key, TRuntimeNode item2) { + Y_UNUSED(key); + return pb.Equals(item2, pb.NewDataLiteral<ui64>(0)); + }); + }, + 1, + pb.NewFlowType(pb.NewDataType(NUdf::EDataSlot::String)) + ); + + const auto pgm = StreamToString(setup, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|"); + } +} +#endif +} // NMiniKQL +} // NKikimr diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp new file mode 100644 index 00000000000..72fe11dc404 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp @@ -0,0 +1,1576 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/mkql_runtime_version.h> +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> + +#include <cstring> +#include <random> +#include <ctime> +#include <algorithm> + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + +ui64 g_Yield = std::numeric_limits<ui64>::max(); +ui64 g_TestStreamData[] = {0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2}; +ui64 g_TestYieldStreamData[] = {0, 1, 2, g_Yield, 0, g_Yield, 1, 2, 0, 1, 2, 0, g_Yield, 1, 2}; + +template <bool WithYields> +class TTestStreamWrapper: public TMutableComputationNode<TTestStreamWrapper<WithYields>> { + typedef TMutableComputationNode<TTestStreamWrapper<WithYields>> TBaseComputation; +public: + class TStreamValue : public TComputationValue<TStreamValue> { + public: + using TBase = TComputationValue<TStreamValue>; + + TStreamValue(TMemoryUsageInfo* memInfo, TComputationContext& compCtx, const TTestStreamWrapper* parent) + : TBase(memInfo) + , CompCtx(compCtx) + , Parent(parent) + { + } + + private: + NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override { + constexpr auto size = WithYields ? Y_ARRAY_SIZE(g_TestYieldStreamData) : Y_ARRAY_SIZE(g_TestStreamData); + if (Index == size) { + return NUdf::EFetchStatus::Finish; + } + + const auto val = WithYields ? g_TestYieldStreamData[Index] : g_TestStreamData[Index]; + if (g_Yield == val) { + ++Index; + return NUdf::EFetchStatus::Yield; + } + + NUdf::TUnboxedValue* items = nullptr; + result = CompCtx.HolderFactory.CreateDirectArrayHolder(2, items); + items[0] = NUdf::TUnboxedValuePod(val); + if (((Index + 1) % Parent->PeakStep) == 0) { + auto str = MakeStringNotFilled(64ul << 20); + const auto& buf = str.AsStringRef(); + memset(buf.Data(), ' ', buf.Size()); + items[1] = std::move(str); + } else { + items[1] = NUdf::TUnboxedValuePod::Zero(); + } + + ++Index; + return NUdf::EFetchStatus::Ok; + } + + private: + TComputationContext& CompCtx; + const TTestStreamWrapper* const Parent; + ui64 Index = 0; + }; + + TTestStreamWrapper(TComputationMutables& mutables, ui64 peakStep) + : TBaseComputation(mutables) + , PeakStep(peakStep) + { + } + + NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { + return ctx.HolderFactory.Create<TStreamValue>(ctx, this); + } + +private: + void RegisterDependencies() const final { + } + +private: + const ui64 PeakStep; +}; + +template <bool WithYields> +IComputationNode* WrapTestStream(TCallable& callable, const TComputationNodeFactoryContext& ctx) { + MKQL_ENSURE(callable.GetInputsCount() == 1, "Expected 1 args"); + const ui64 peakStep = AS_VALUE(TDataLiteral, callable.GetInput(0))->AsValue().Get<ui64>(); + return new TTestStreamWrapper<WithYields>(ctx.Mutables, peakStep); +} + +TIntrusivePtr<IRandomProvider> CreateRandomProvider() { + return CreateDeterministicRandomProvider(1); +} + +TIntrusivePtr<ITimeProvider> CreateTimeProvider() { + return CreateDeterministicTimeProvider(10000000); +} + +TComputationNodeFactory GetTestFactory() { + return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* { + if (callable.GetType()->GetName() == "TestList") { + return new TExternalComputationNode(ctx.Mutables); + } + if (callable.GetType()->GetName() == "TestStream") { + return WrapTestStream<false>(callable, ctx); + } + if (callable.GetType()->GetName() == "TestYieldStream") { + return WrapTestStream<true>(callable, ctx); + } + return GetBuiltinFactory()(callable, ctx); + }; +} + +template<bool UseLLVM> +struct TSetup_ { + TSetup_() + : Alloc(__LOCATION__) + { + FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); + RandomProvider = CreateRandomProvider(); + TimeProvider = CreateTimeProvider(); + + Env.Reset(new TTypeEnvironment(Alloc)); + PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry)); + } + + TAutoPtr<IComputationGraph> BuildGraph(TRuntimeNode pgm, EGraphPerProcess graphPerProcess = EGraphPerProcess::Multi, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) { + Explorer.Walk(pgm.GetNode(), *Env); + TComputationPatternOpts opts(Alloc.Ref(), *Env, GetTestFactory(), FunctionRegistry.Get(), + NUdf::EValidateMode::None, NUdf::EValidatePolicy::Exception, UseLLVM ? "" : "OFF", graphPerProcess); + Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts); + return Pattern->Clone(opts.ToComputationOptions(*RandomProvider, *TimeProvider)); + } + + TIntrusivePtr<IFunctionRegistry> FunctionRegistry; + TIntrusivePtr<IRandomProvider> RandomProvider; + TIntrusivePtr<ITimeProvider> TimeProvider; + + TScopedAlloc Alloc; + THolder<TTypeEnvironment> Env; + THolder<TProgramBuilder> PgmBuilder; + + TExploringNodeVisitor Explorer; + IComputationPattern::TPtr Pattern; +}; + +template <bool LLVM, bool WithYields = false> +TRuntimeNode MakeStream(TSetup_<LLVM>& setup, ui64 peakStep) { + TProgramBuilder& pb = *setup.PgmBuilder; + + TCallableBuilder callableBuilder(*setup.Env, WithYields ? "TestYieldStream" : "TestStream", + pb.NewStreamType( + pb.NewStructType({ + {TStringBuf("a"), pb.NewDataType(NUdf::EDataSlot::Uint64)}, + {TStringBuf("b"), pb.NewDataType(NUdf::EDataSlot::String)} + }) + ) + ); + callableBuilder.Add(pb.NewDataLiteral(peakStep)); + + return TRuntimeNode(callableBuilder.Build(), false); +} + +template <bool OverFlow> +TRuntimeNode Combine(TProgramBuilder& pb, TRuntimeNode stream, std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)> finishLambda) { + const auto keyExtractor = [&](TRuntimeNode item) { + return pb.Member(item, "a"); + }; + const auto init = [&](TRuntimeNode /*key*/, TRuntimeNode item) { + return item; + }; + const auto update = [&](TRuntimeNode /*key*/, TRuntimeNode item, TRuntimeNode state) { + const auto a = pb.Add(pb.Member(item, "a"), pb.Member(state, "a")); + const auto b = pb.Concat(pb.Member(item, "b"), pb.Member(state, "b")); + return pb.NewStruct({ + {TStringBuf("a"), a}, + {TStringBuf("b"), b}, + }); + }; + + return OverFlow ? + pb.FromFlow(pb.CombineCore(pb.ToFlow(stream), keyExtractor, init, update, finishLambda, 64ul << 20)): + pb.CombineCore(stream, keyExtractor, init, update, finishLambda, 64ul << 20); +} + +TRuntimeNode Reduce(TProgramBuilder& pb, TRuntimeNode stream) { + return pb.Condense(stream, pb.NewDataLiteral<ui64>(0), + [&] (TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral<bool>(false); }, + [&] (TRuntimeNode item, TRuntimeNode state) { return pb.Add(state, item); } + ); +} + +TRuntimeNode StreamToString(TProgramBuilder& pb, TRuntimeNode stream) { + const auto sorted = pb.Sort(stream, pb.NewDataLiteral(true), + [&](TRuntimeNode item) { + return item; + }); + + return pb.Condense(sorted, pb.NewDataLiteral<NUdf::EDataSlot::String>("|"), + [&] (TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral<bool>(false); }, + [&] (TRuntimeNode item, TRuntimeNode state) { + return pb.Concat(pb.Concat(state, pb.ToString(item)), pb.NewDataLiteral<NUdf::EDataSlot::String>("|")); + } + ); +} + +} // unnamed + +Y_UNIT_TEST_SUITE(TMiniKQLCombineStreamTest) { + Y_UNIT_TEST_LLVM(TestFullCombineWithOptOut) { + TSetup_<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) { + return pb.NewOptional(pb.Member(state, "a")); + }; + + const auto stream = MakeStream(setup, Max<ui64>()); + const auto pgm = StreamToString(pb, Combine<false>(pb, stream, finish)); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|4|8|"); + } + + Y_UNIT_TEST_LLVM(TestFullCombineWithListOut) { + TSetup_<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) { + const auto item = pb.Member(state, "a"); + const auto itemType = item.GetStaticType(); + auto list = pb.NewEmptyList(itemType); + list = pb.Append(list, item); + list = pb.Append(list, item); + return list; + }; + + const auto stream = MakeStream(setup, Max<ui64>()); + const auto pgm = StreamToString(pb, Combine<false>(pb, stream, finish)); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|4|4|8|8|"); + } + + Y_UNIT_TEST_LLVM(TestFullCombineWithStreamOut) { + TSetup_<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) { + const auto item = pb.Member(state, "a"); + const auto itemType = item.GetStaticType(); + auto list = pb.NewEmptyList(itemType); + list = pb.Append(list, item); + list = pb.Append(list, item); + return pb.Iterator(list, MakeArrayRef(&state, 1)); + }; + + const auto stream = MakeStream(setup, Max<ui64>()); + const auto pgm = StreamToString(pb, Combine<false>(pb, stream, finish)); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|4|4|8|8|"); + } + + Y_UNIT_TEST_LLVM(TestFullCombineWithOptOutAndYields) { + TSetup_<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) { + return pb.NewOptional(pb.Member(state, "a")); + }; + + const auto stream = MakeStream<LLVM, true>(setup, Max<ui64>()); + const auto pgm = StreamToString(pb, Combine<false>(pb, stream, finish)); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|1|1|2|2|2|4|"); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Finish); + } + + Y_UNIT_TEST_LLVM(TestFullCombineWithListAndYields) { + TSetup_<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) { + const auto item = pb.Member(state, "a"); + const auto itemType = item.GetStaticType(); + auto list = pb.NewEmptyList(itemType); + list = pb.Append(list, item); + list = pb.Append(list, item); + return list; + }; + + const auto stream = MakeStream<LLVM, true>(setup, Max<ui64>()); + const auto pgm = StreamToString(pb, Combine<false>(pb, stream, finish)); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|0|1|1|1|1|2|2|2|2|2|2|4|4|"); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Finish); + } + + Y_UNIT_TEST_LLVM(TestFullCombineWithStreamAndYields) { + TSetup_<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) { + const auto item = pb.Member(state, "a"); + const auto itemType = item.GetStaticType(); + auto list = pb.NewEmptyList(itemType); + list = pb.Append(list, item); + list = pb.Append(list, item); + return pb.Iterator(list, MakeArrayRef(&state, 1)); + }; + + const auto stream = MakeStream<LLVM, true>(setup, Max<ui64>()); + const auto pgm = StreamToString(pb, Combine<false>(pb, stream, finish)); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|0|1|1|1|1|2|2|2|2|2|2|4|4|"); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Finish); + } + + Y_UNIT_TEST_LLVM(TestPartialFlush) { + TSetup_<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) { + return pb.NewOptional(pb.Member(state, "a")); + }; + + const auto stream = MakeStream(setup, 6ul); + const auto combine = Combine<false>(pb, stream, finish); + { + const auto pgm = Reduce(pb, combine); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(result.Get<ui64>(), 12ul); + } + { + const auto pgm = StreamToString(pb, combine); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|2|2|4|4|"); + } + } + + Y_UNIT_TEST_LLVM(TestCombineInSingleProc) { + TSetup_<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) { + return pb.NewOptional(pb.Member(state, "a")); + }; + + const auto stream = MakeStream(setup, 6ul); + const auto pgm = Reduce(pb, Combine<false>(pb, stream, finish)); + const auto graph = setup.BuildGraph(pgm, EGraphPerProcess::Single); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(result.Get<ui64>(), 12ul); + } + + Y_UNIT_TEST_LLVM(TestCombineSwithYield) { + TSetup_<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) { + return pb.NewOptional(pb.Member(state, "a")); + }; + + auto stream = MakeStream(setup, Max<ui64>()); + TSwitchInput switchInput; + switchInput.Indicies.push_back(0); + switchInput.InputType = stream.GetStaticType(); + + stream = pb.Switch(stream, + MakeArrayRef(&switchInput, 1), + [&](ui32 /*index*/, TRuntimeNode item) { return Combine<false>(pb, item, finish); }, + 1, + pb.NewStreamType(pb.NewDataType(NUdf::EDataSlot::Uint64)) + ); + + const auto pgm = StreamToString(pb, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|1|1|1|1|2|2|2|2|"); + } +} + +Y_UNIT_TEST_SUITE(TMiniKQLCombineStreamPerfTest) { + Y_UNIT_TEST_LLVM(TestSumDoubleBooleanKeys) { + TSetup_<LLVM> setup; + + double positive = 0.0, negative = 0.0; + const auto t = TInstant::Now(); + for (const auto& sample : I8Samples) { + (sample.second > 0.0 ? positive : negative) += sample.second; + } + const auto cppTime = TInstant::Now() - t; + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.CombineCore(pb.Iterator(TRuntimeNode(list, false), {}), + [&](TRuntimeNode item) { return pb.AggrGreater(item, pb.NewDataLiteral(0.0)); }, + [&](TRuntimeNode, TRuntimeNode item) { return item; }, + [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, item); }, + [&](TRuntimeNode, TRuntimeNode state) { return pb.NewOptional(state); }, + 0ULL + ); + + const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items)); + std::transform(I8Samples.cbegin(), I8Samples.cend(), items, [](const std::pair<i8, double> s){ return ToValue<double>(s.second); }); + + NUdf::TUnboxedValue first, second; + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + UNIT_ASSERT_EQUAL(value.Fetch(first), NUdf::EFetchStatus::Ok); + UNIT_ASSERT_EQUAL(value.Fetch(second), NUdf::EFetchStatus::Ok); + const auto t2 = TInstant::Now(); + + if (first.template Get<double>() > 0.0) { + UNIT_ASSERT_VALUES_EQUAL(first.template Get<double>(), positive); + UNIT_ASSERT_VALUES_EQUAL(second.template Get<double>(), negative); + } else { + UNIT_ASSERT_VALUES_EQUAL(first.template Get<double>(), negative); + UNIT_ASSERT_VALUES_EQUAL(second.template Get<double>(), positive); + } + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleBooleanKeys) { + TSetup_<LLVM> setup; + + double pSum = 0.0, nSum = 0.0, pMax = 0.0, nMax = -1000.0, pMin = 1000.0, nMin = 0.0; + const auto t = TInstant::Now(); + for (const auto& sample : I8Samples) { + if (sample.second > 0.0) { + pSum += sample.second; + pMax = std::max(pMax, sample.second); + pMin = std::min(pMin, sample.second); + } else { + nSum += sample.second; + nMax = std::max(nMax, sample.second); + nMin = std::min(nMin, sample.second); + } + } + + const auto cppTime = TInstant::Now() - t; + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.CombineCore(pb.Iterator(TRuntimeNode(list, false), {}), + [&](TRuntimeNode item) { return pb.AggrGreater(item, pb.NewDataLiteral(0.0)); }, + [&](TRuntimeNode, TRuntimeNode item) { return pb.NewTuple({item, item, item}); }, + [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0U), item), pb.AggrMin(pb.Nth(state, 1U), item), pb.AggrMax(pb.Nth(state, 2U), item) }); }, + [&](TRuntimeNode, TRuntimeNode state) { return pb.NewOptional(state); }, + 0ULL + ); + + const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items)); + std::transform(I8Samples.cbegin(), I8Samples.cend(), items, [](const std::pair<i8, double> s){ return ToValue<double>(s.second); }); + + NUdf::TUnboxedValue first, second; + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + UNIT_ASSERT_EQUAL(value.Fetch(first), NUdf::EFetchStatus::Ok); + UNIT_ASSERT_EQUAL(value.Fetch(second), NUdf::EFetchStatus::Ok); + const auto t2 = TInstant::Now(); + + if (first.GetElement(0).template Get<double>() > 0.0) { + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(0).template Get<double>(), pSum); + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(1).template Get<double>(), pMin); + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(2).template Get<double>(), pMax); + + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(0).template Get<double>(), nSum); + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(1).template Get<double>(), nMin); + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(2).template Get<double>(), nMax); + } else { + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(0).template Get<double>(), nSum); + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(1).template Get<double>(), nMin); + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(2).template Get<double>(), nMax); + + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(0).template Get<double>(), pSum); + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(1).template Get<double>(), pMin); + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(2).template Get<double>(), pMax); + } + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestSumDoubleSmallKey) { + TSetup_<LLVM> setup; + + std::unordered_map<i8, double> expects(201); + const auto t = TInstant::Now(); + for (const auto& sample : I8Samples) { + expects.emplace(sample.first, 0.0).first->second += sample.second; + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<i8, double>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<i8, double> l, const std::pair<i8, double> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i8>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.CombineCore(pb.Iterator(TRuntimeNode(list, false), {}), + [&](TRuntimeNode item) { return pb.Nth(item, 0U); }, + [&](TRuntimeNode, TRuntimeNode item) { return pb.Nth(item, 1U); }, + [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, pb.Nth(item, 1U)); }, + [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, state})); }, + 0ULL + )); + + const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items)); + for (const auto& sample : I8Samples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[0] = NUdf::TUnboxedValuePod(sample.first); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + two.emplace_back(ptr[i].GetElement(0).template Get<i8>(), ptr[i].GetElement(1).template Get<double>()); + } + + std::sort(two.begin(), two.end(), [](const std::pair<i8, double> l, const std::pair<i8, double> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleSmallKey) { + TSetup_<LLVM> setup; + + std::unordered_map<i8, std::array<double, 3U>> expects(201); + const auto t = TInstant::Now(); + for (const auto& sample : I8Samples) { + auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, std::numeric_limits<double>::max(), std::numeric_limits<double>::min()}).first->second; + std::get<0U>(item) += sample.second; + std::get<1U>(item) = std::min(std::get<1U>(item), sample.second); + std::get<2U>(item) = std::max(std::get<2U>(item), sample.second); + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<i8, std::array<double, 3U>>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<i8, std::array<double, 3U>> l, const std::pair<i8, std::array<double, 3U>> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i8>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.CombineCore(pb.Iterator(TRuntimeNode(list, false), {}), + [&](TRuntimeNode item) { return pb.Nth(item, 0U); }, + [&](TRuntimeNode, TRuntimeNode item) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({v, v, v}); }, + [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0U), v), pb.AggrMin(pb.Nth(state, 1U), v), pb.AggrMax(pb.Nth(state, 2U), v)}); }, + [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, pb.Nth(state, 0U), pb.Nth(state, 1U), pb.Nth(state, 2U)})); }, + 0ULL + )); + + const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items)); + for (const auto& sample : I8Samples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[0] = NUdf::TUnboxedValuePod(sample.first); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + two.emplace_back(ptr[i].GetElement(0).template Get<i8>(), std::array<double, 3U>{ptr[i].GetElement(1).template Get<double>(), ptr[i].GetElement(2).template Get<double>(), ptr[i].GetElement(3).template Get<double>()}); + } + + std::sort(two.begin(), two.end(), [](const std::pair<i8, std::array<double, 3U>> l, const std::pair<i8, std::array<double, 3U>> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestSumDoubleStringKey) { + TSetup_<LLVM> setup; + + std::vector<std::pair<std::string, double>> stringI8Samples(I8Samples.size()); + std::transform(I8Samples.cbegin(), I8Samples.cend(), stringI8Samples.begin(), [](std::pair<i8, double> src){ return std::make_pair(ToString(src.first), src.second); }); + + std::unordered_map<std::string, double> expects(201); + const auto t = TInstant::Now(); + for (const auto& sample : stringI8Samples) { + expects.emplace(sample.first, 0.0).first->second += sample.second; + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<std::string_view, double>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<std::string_view, double> l, const std::pair<std::string_view, double> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<const char*>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.CombineCore(pb.Iterator(TRuntimeNode(list, false), {}), + [&](TRuntimeNode item) { return pb.Nth(item, 0U); }, + [&](TRuntimeNode, TRuntimeNode item) { return pb.Nth(item, 1U); }, + [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, pb.Nth(item, 1U)); }, + [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, state})); }, + 0ULL + )); + + const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(stringI8Samples.size(), items)); + for (const auto& sample : stringI8Samples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[0] = NUdf::TUnboxedValuePod::Embedded(sample.first); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + two.emplace_back(ptr[i].GetElements()->AsStringRef(), ptr[i].GetElement(1).template Get<double>()); + } + + std::sort(two.begin(), two.end(), [](const std::pair<std::string_view, double> l, const std::pair<std::string_view, double> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleStringKey) { + TSetup_<LLVM> setup; + + std::vector<std::pair<std::string, double>> stringI8Samples(I8Samples.size()); + std::transform(I8Samples.cbegin(), I8Samples.cend(), stringI8Samples.begin(), [](std::pair<i8, double> src){ return std::make_pair(ToString(src.first), src.second); }); + + std::unordered_map<std::string, std::array<double, 3U>> expects(201); + const auto t = TInstant::Now(); + for (const auto& sample : stringI8Samples) { + auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, +1E7, -1E7}).first->second; + std::get<0U>(item) += sample.second; + std::get<1U>(item) = std::min(std::get<1U>(item), sample.second); + std::get<2U>(item) = std::max(std::get<2U>(item), sample.second); + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<std::string_view, std::array<double, 3U>>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<std::string_view, std::array<double, 3U>> l, const std::pair<std::string_view, std::array<double, 3U>> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<const char*>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.CombineCore(pb.Iterator(TRuntimeNode(list, false), {}), + [&](TRuntimeNode item) { return pb.Nth(item, 0U); }, + [&](TRuntimeNode, TRuntimeNode item) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({v, v, v}); }, + [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0U), v), pb.AggrMin(pb.Nth(state, 1U), v), pb.AggrMax(pb.Nth(state, 2U), v)}); }, + [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, pb.Nth(state, 0U), pb.Nth(state, 1U), pb.Nth(state, 2U)})); }, + 0ULL + )); + + const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(stringI8Samples.size(), items)); + for (const auto& sample : stringI8Samples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[0] = NUdf::TUnboxedValuePod::Embedded(sample.first); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + two.emplace_back(ptr[i].GetElements()->AsStringRef(), std::array<double, 3U>{ptr[i].GetElement(1).template Get<double>(), ptr[i].GetElement(2).template Get<double>(), ptr[i].GetElement(3).template Get<double>()}); + } + + std::sort(two.begin(), two.end(), [](const std::pair<std::string_view, std::array<double, 3U>> l, const std::pair<std::string_view, std::array<double, 3U>> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumTupleKey) { + TSetup_<LLVM> setup; + + std::vector<std::pair<std::pair<ui32, std::string>, double>> pairI8Samples(Ui16Samples.size()); + std::transform(Ui16Samples.cbegin(), Ui16Samples.cend(), pairI8Samples.begin(), [](std::pair<ui32, double> src){ return std::make_pair(std::make_pair(ui32(src.first / 10U % 100U), ToString(src.first % 10U)), src.second); }); + + struct TPairHash { size_t operator()(const std::pair<ui16, std::string>& p) const { return CombineHashes(std::hash<ui32>()(p.first), std::hash<std::string_view>()(p.second)); } }; + + std::unordered_map<std::pair<ui32, std::string>, std::array<double, 3U>, TPairHash> expects; + const auto t = TInstant::Now(); + for (const auto& sample : pairI8Samples) { + auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, +1E7, -1E7}).first->second; + std::get<0U>(item) += sample.second; + std::get<1U>(item) = std::min(std::get<1U>(item), sample.second); + std::get<2U>(item) = std::max(std::get<2U>(item), sample.second); + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<std::pair<ui32, std::string>, std::array<double, 3U>>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> l, const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<const char*>::Id)}), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.CombineCore(pb.Iterator(TRuntimeNode(list, false), {}), + [&](TRuntimeNode item) { return pb.Nth(item, 0U); }, + [&](TRuntimeNode, TRuntimeNode item) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({v, v, v}); }, + [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0U), v), pb.AggrMin(pb.Nth(state, 1U), v), pb.AggrMax(pb.Nth(state, 2U), v)}); }, + [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, pb.Nth(state, 0U), pb.Nth(state, 1U), pb.Nth(state, 2U)})); }, + 0ULL + )); + + const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(pairI8Samples.size(), items)); + for (const auto& sample : pairI8Samples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + NUdf::TUnboxedValue* keys = nullptr; + pair[0] = graph->GetHolderFactory().CreateDirectArrayHolder(2U, keys); + keys[0] = NUdf::TUnboxedValuePod(sample.first.first); + keys[1] = NUdf::TUnboxedValuePod::Embedded(sample.first.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + const auto elements = ptr[i].GetElements(); + two.emplace_back(std::make_pair(elements[0].GetElement(0).template Get<ui32>(), (elements[0].GetElements()[1]).AsStringRef()), std::array<double, 3U>{elements[1].template Get<double>(), elements[2].template Get<double>(), elements[3].template Get<double>()}); + } + + std::sort(two.begin(), two.end(), [](const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> l, const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } +} +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 3u +Y_UNIT_TEST_SUITE(TMiniKQLCombineFlowTest) { + Y_UNIT_TEST_LLVM(TestFullCombineWithOptOut) { + TSetup_<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) { + return pb.NewOptional(pb.Member(state, "a")); + }; + + const auto stream = MakeStream(setup, Max<ui64>()); + const auto pgm = StreamToString(pb, Combine<true>(pb, stream, finish)); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|4|8|"); + } + + Y_UNIT_TEST_LLVM(TestFullCombineWithListOut) { + TSetup_<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) { + const auto item = pb.Member(state, "a"); + const auto itemType = item.GetStaticType(); + auto list = pb.NewEmptyList(itemType); + list = pb.Append(list, item); + list = pb.Append(list, item); + return list; + }; + + const auto stream = MakeStream(setup, Max<ui64>()); + const auto pgm = StreamToString(pb, Combine<true>(pb, stream, finish)); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|4|4|8|8|"); + } + + Y_UNIT_TEST_LLVM(TestFullCombineWithStreamOut) { + TSetup_<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) { + const auto item = pb.Member(state, "a"); + const auto itemType = item.GetStaticType(); + auto list = pb.NewEmptyList(itemType); + list = pb.Append(list, item); + list = pb.Append(list, item); + return pb.Iterator(list, MakeArrayRef(&state, 1)); + }; + + const auto stream = MakeStream(setup, Max<ui64>()); + const auto pgm = StreamToString(pb, Combine<true>(pb, stream, finish)); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|4|4|8|8|"); + } + + Y_UNIT_TEST_LLVM(TestFullCombineWithOptOutAndYields) { + TSetup_<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) { + return pb.NewOptional(pb.Member(state, "a")); + }; + + const auto stream = MakeStream<LLVM, true>(setup, Max<ui64>()); + const auto pgm = StreamToString(pb, Combine<true>(pb, stream, finish)); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|1|1|2|2|2|4|"); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Finish); + } + + Y_UNIT_TEST_LLVM(TestFullCombineWithListAndYields) { + TSetup_<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) { + const auto item = pb.Member(state, "a"); + const auto itemType = item.GetStaticType(); + auto list = pb.NewEmptyList(itemType); + list = pb.Append(list, item); + list = pb.Append(list, item); + return list; + }; + + const auto stream = MakeStream<LLVM, true>(setup, Max<ui64>()); + const auto pgm = StreamToString(pb, Combine<true>(pb, stream, finish)); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|0|1|1|1|1|2|2|2|2|2|2|4|4|"); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Finish); + } + + Y_UNIT_TEST_LLVM(TestFullCombineWithStreamAndYields) { + TSetup_<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) { + const auto item = pb.Member(state, "a"); + const auto itemType = item.GetStaticType(); + auto list = pb.NewEmptyList(itemType); + list = pb.Append(list, item); + list = pb.Append(list, item); + return pb.Iterator(list, MakeArrayRef(&state, 1)); + }; + + const auto stream = MakeStream<LLVM, true>(setup, Max<ui64>()); + const auto pgm = StreamToString(pb, Combine<true>(pb, stream, finish)); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|0|1|1|1|1|2|2|2|2|2|2|4|4|"); + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Finish); + } + + Y_UNIT_TEST_LLVM(TestPartialFlush) { + TSetup_<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) { + return pb.NewOptional(pb.Member(state, "a")); + }; + + const auto stream = MakeStream(setup, 6ul); + const auto combine = Combine<true>(pb, stream, finish); + { + const auto pgm = Reduce(pb, combine); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(result.Get<ui64>(), 12ul); + } + { + const auto pgm = StreamToString(pb, combine); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|2|2|4|4|"); + } + } + + Y_UNIT_TEST_LLVM(TestCombineInSingleProc) { + TSetup_<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) { + return pb.NewOptional(pb.Member(state, "a")); + }; + + const auto stream = MakeStream(setup, 6ul); + const auto pgm = Reduce(pb, Combine<true>(pb, stream, finish)); + const auto graph = setup.BuildGraph(pgm, EGraphPerProcess::Single); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(result.Get<ui64>(), 12ul); + } + + Y_UNIT_TEST_LLVM(TestCombineSwithYield) { + TSetup_<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) { + return pb.NewOptional(pb.Member(state, "a")); + }; + + auto stream = MakeStream(setup, Max<ui64>()); + TSwitchInput switchInput; + switchInput.Indicies.push_back(0); + switchInput.InputType = stream.GetStaticType(); + + stream = pb.Switch(stream, + MakeArrayRef(&switchInput, 1), + [&](ui32 /*index*/, TRuntimeNode item) { return Combine<true>(pb, item, finish); }, + 1, + pb.NewStreamType(pb.NewDataType(NUdf::EDataSlot::Uint64)) + ); + + const auto pgm = StreamToString(pb, stream); + const auto graph = setup.BuildGraph(pgm); + const auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|1|1|1|1|2|2|2|2|"); + } +} + +Y_UNIT_TEST_SUITE(TMiniKQLCombineFlowPerfTest) { + Y_UNIT_TEST_LLVM(TestSumDoubleBooleanKeys) { + TSetup_<LLVM> setup; + + double positive = 0.0, negative = 0.0; + const auto t = TInstant::Now(); + for (const auto& sample : I8Samples) { + (sample.second > 0.0 ? positive : negative) += sample.second; + } + const auto cppTime = TInstant::Now() - t; + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.FromFlow(pb.CombineCore(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) { return pb.AggrGreater(item, pb.NewDataLiteral(0.0)); }, + [&](TRuntimeNode, TRuntimeNode item) { return item; }, + [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, item); }, + [&](TRuntimeNode, TRuntimeNode state) { return pb.NewOptional(state); }, + 0ULL + )); + + const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items)); + std::transform(I8Samples.cbegin(), I8Samples.cend(), items, [](const std::pair<i8, double> s){ return ToValue<double>(s.second); }); + + NUdf::TUnboxedValue first, second; + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + UNIT_ASSERT_EQUAL(value.Fetch(first), NUdf::EFetchStatus::Ok); + UNIT_ASSERT_EQUAL(value.Fetch(second), NUdf::EFetchStatus::Ok); + const auto t2 = TInstant::Now(); + + if (first.template Get<double>() > 0.0) { + UNIT_ASSERT_VALUES_EQUAL(first.template Get<double>(), positive); + UNIT_ASSERT_VALUES_EQUAL(second.template Get<double>(), negative); + } else { + UNIT_ASSERT_VALUES_EQUAL(first.template Get<double>(), negative); + UNIT_ASSERT_VALUES_EQUAL(second.template Get<double>(), positive); + } + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleBooleanKeys) { + TSetup_<LLVM> setup; + + double pSum = 0.0, nSum = 0.0, pMax = 0.0, nMax = -1000.0, pMin = 1000.0, nMin = 0.0; + const auto t = TInstant::Now(); + for (const auto& sample : I8Samples) { + if (sample.second > 0.0) { + pSum += sample.second; + pMax = std::max(pMax, sample.second); + pMin = std::min(pMin, sample.second); + } else { + nSum += sample.second; + nMax = std::max(nMax, sample.second); + nMin = std::min(nMin, sample.second); + } + } + + const auto cppTime = TInstant::Now() - t; + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.FromFlow(pb.CombineCore(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) { return pb.AggrGreater(item, pb.NewDataLiteral(0.0)); }, + [&](TRuntimeNode, TRuntimeNode item) { return pb.NewTuple({item, item, item}); }, + [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0U), item), pb.AggrMin(pb.Nth(state, 1U), item), pb.AggrMax(pb.Nth(state, 2U), item) }); }, + [&](TRuntimeNode, TRuntimeNode state) { return pb.NewOptional(state); }, + 0ULL + )); + + const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items)); + std::transform(I8Samples.cbegin(), I8Samples.cend(), items, [](const std::pair<i8, double> s){ return ToValue<double>(s.second); }); + + NUdf::TUnboxedValue first, second; + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + UNIT_ASSERT_EQUAL(value.Fetch(first), NUdf::EFetchStatus::Ok); + UNIT_ASSERT_EQUAL(value.Fetch(second), NUdf::EFetchStatus::Ok); + const auto t2 = TInstant::Now(); + + if (first.GetElement(0).template Get<double>() > 0.0) { + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(0).template Get<double>(), pSum); + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(1).template Get<double>(), pMin); + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(2).template Get<double>(), pMax); + + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(0).template Get<double>(), nSum); + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(1).template Get<double>(), nMin); + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(2).template Get<double>(), nMax); + } else { + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(0).template Get<double>(), nSum); + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(1).template Get<double>(), nMin); + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(2).template Get<double>(), nMax); + + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(0).template Get<double>(), pSum); + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(1).template Get<double>(), pMin); + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(2).template Get<double>(), pMax); + } + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestSumDoubleSmallKey) { + TSetup_<LLVM> setup; + + std::unordered_map<i8, double> expects(201); + const auto t = TInstant::Now(); + for (const auto& sample : I8Samples) { + expects.emplace(sample.first, 0.0).first->second += sample.second; + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<i8, double>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<i8, double> l, const std::pair<i8, double> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i8>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.CombineCore(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) { return pb.Nth(item, 0U); }, + [&](TRuntimeNode, TRuntimeNode item) { return pb.Nth(item, 1U); }, + [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, pb.Nth(item, 1U)); }, + [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, state})); }, + 0ULL + )); + + const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items)); + for (const auto& sample : I8Samples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[0] = NUdf::TUnboxedValuePod(sample.first); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + two.emplace_back(ptr[i].GetElement(0).template Get<i8>(), ptr[i].GetElement(1).template Get<double>()); + } + + std::sort(two.begin(), two.end(), [](const std::pair<i8, double> l, const std::pair<i8, double> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleSmallKey) { + TSetup_<LLVM> setup; + + std::unordered_map<i8, std::array<double, 3U>> expects(201); + const auto t = TInstant::Now(); + for (const auto& sample : I8Samples) { + auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, std::numeric_limits<double>::max(), std::numeric_limits<double>::min()}).first->second; + std::get<0U>(item) += sample.second; + std::get<1U>(item) = std::min(std::get<1U>(item), sample.second); + std::get<2U>(item) = std::max(std::get<2U>(item), sample.second); + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<i8, std::array<double, 3U>>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<i8, std::array<double, 3U>> l, const std::pair<i8, std::array<double, 3U>> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i8>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.CombineCore(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) { return pb.Nth(item, 0U); }, + [&](TRuntimeNode, TRuntimeNode item) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({v, v, v}); }, + [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0U), v), pb.AggrMin(pb.Nth(state, 1U), v), pb.AggrMax(pb.Nth(state, 2U), v)}); }, + [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, pb.Nth(state, 0U), pb.Nth(state, 1U), pb.Nth(state, 2U)})); }, + 0ULL + )); + + const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items)); + for (const auto& sample : I8Samples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[0] = NUdf::TUnboxedValuePod(sample.first); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + two.emplace_back(ptr[i].GetElement(0).template Get<i8>(), std::array<double, 3U>{ptr[i].GetElement(1).template Get<double>(), ptr[i].GetElement(2).template Get<double>(), ptr[i].GetElement(3).template Get<double>()}); + } + + std::sort(two.begin(), two.end(), [](const std::pair<i8, std::array<double, 3U>> l, const std::pair<i8, std::array<double, 3U>> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestSumDoubleStringKey) { + TSetup_<LLVM> setup; + + std::vector<std::pair<std::string, double>> stringI8Samples(I8Samples.size()); + std::transform(I8Samples.cbegin(), I8Samples.cend(), stringI8Samples.begin(), [](std::pair<i8, double> src){ return std::make_pair(ToString(src.first), src.second); }); + + std::unordered_map<std::string, double> expects(201); + const auto t = TInstant::Now(); + for (const auto& sample : stringI8Samples) { + expects.emplace(sample.first, 0.0).first->second += sample.second; + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<std::string_view, double>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<std::string_view, double> l, const std::pair<std::string_view, double> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<const char*>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.CombineCore(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) { return pb.Nth(item, 0U); }, + [&](TRuntimeNode, TRuntimeNode item) { return pb.Nth(item, 1U); }, + [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, pb.Nth(item, 1U)); }, + [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, state})); }, + 0ULL + )); + + const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(stringI8Samples.size(), items)); + for (const auto& sample : stringI8Samples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[0] = NUdf::TUnboxedValuePod::Embedded(sample.first); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + two.emplace_back(ptr[i].GetElements()->AsStringRef(), ptr[i].GetElement(1).template Get<double>()); + } + + std::sort(two.begin(), two.end(), [](const std::pair<std::string_view, double> l, const std::pair<std::string_view, double> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleStringKey) { + TSetup_<LLVM> setup; + + std::vector<std::pair<std::string, double>> stringI8Samples(I8Samples.size()); + std::transform(I8Samples.cbegin(), I8Samples.cend(), stringI8Samples.begin(), [](std::pair<i8, double> src){ return std::make_pair(ToString(src.first), src.second); }); + + std::unordered_map<std::string, std::array<double, 3U>> expects(201); + const auto t = TInstant::Now(); + for (const auto& sample : stringI8Samples) { + auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, +1E7, -1E7}).first->second; + std::get<0U>(item) += sample.second; + std::get<1U>(item) = std::min(std::get<1U>(item), sample.second); + std::get<2U>(item) = std::max(std::get<2U>(item), sample.second); + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<std::string_view, std::array<double, 3U>>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<std::string_view, std::array<double, 3U>> l, const std::pair<std::string_view, std::array<double, 3U>> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<const char*>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.CombineCore(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) { return pb.Nth(item, 0U); }, + [&](TRuntimeNode, TRuntimeNode item) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({v, v, v}); }, + [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0U), v), pb.AggrMin(pb.Nth(state, 1U), v), pb.AggrMax(pb.Nth(state, 2U), v)}); }, + [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, pb.Nth(state, 0U), pb.Nth(state, 1U), pb.Nth(state, 2U)})); }, + 0ULL + )); + + const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(stringI8Samples.size(), items)); + for (const auto& sample : stringI8Samples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[0] = NUdf::TUnboxedValuePod::Embedded(sample.first); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + two.emplace_back(ptr[i].GetElements()->AsStringRef(), std::array<double, 3U>{ptr[i].GetElement(1).template Get<double>(), ptr[i].GetElement(2).template Get<double>(), ptr[i].GetElement(3).template Get<double>()}); + } + + std::sort(two.begin(), two.end(), [](const std::pair<std::string_view, std::array<double, 3U>> l, const std::pair<std::string_view, std::array<double, 3U>> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumTupleKey) { + TSetup_<LLVM> setup; + + std::vector<std::pair<std::pair<ui32, std::string>, double>> pairI8Samples(Ui16Samples.size()); + std::transform(Ui16Samples.cbegin(), Ui16Samples.cend(), pairI8Samples.begin(), [](std::pair<ui16, double> src){ return std::make_pair(std::make_pair(ui32(src.first / 10U % 100U), ToString(src.first % 10U)), src.second); }); + + struct TPairHash { size_t operator()(const std::pair<ui32, std::string>& p) const { return CombineHashes(std::hash<ui32>()(p.first), std::hash<std::string_view>()(p.second)); } }; + + std::unordered_map<std::pair<ui32, std::string>, std::array<double, 3U>, TPairHash> expects; + const auto t = TInstant::Now(); + for (const auto& sample : pairI8Samples) { + auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, +1E7, -1E7}).first->second; + std::get<0U>(item) += sample.second; + std::get<1U>(item) = std::min(std::get<1U>(item), sample.second); + std::get<2U>(item) = std::max(std::get<2U>(item), sample.second); + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<std::pair<ui32, std::string>, std::array<double, 3U>>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> l, const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<const char*>::Id)}), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.CombineCore(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) { return pb.Nth(item, 0U); }, + [&](TRuntimeNode, TRuntimeNode item) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({v, v, v}); }, + [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0U), v), pb.AggrMin(pb.Nth(state, 1U), v), pb.AggrMax(pb.Nth(state, 2U), v)}); }, + [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, pb.Nth(state, 0U), pb.Nth(state, 1U), pb.Nth(state, 2U)})); }, + 0ULL + )); + + const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(pairI8Samples.size(), items)); + for (const auto& sample : pairI8Samples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + NUdf::TUnboxedValue* keys = nullptr; + pair[0] = graph->GetHolderFactory().CreateDirectArrayHolder(2U, keys); + keys[0] = NUdf::TUnboxedValuePod(sample.first.first); + keys[1] = NUdf::TUnboxedValuePod::Embedded(sample.first.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + const auto elements = ptr[i].GetElements(); + two.emplace_back(std::make_pair(elements[0].GetElement(0).template Get<ui32>(), (elements[0].GetElements()[1]).AsStringRef()), std::array<double, 3U>{elements[1].template Get<double>(), elements[2].template Get<double>(), elements[3].template Get<double>()}); + } + + std::sort(two.begin(), two.end(), [](const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> l, const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + const auto border = 9124596000000000ULL; + + Y_UNIT_TEST_LLVM(TestTpch) { + TSetup_<LLVM> setup; + + struct TPairHash { size_t operator()(const std::pair<std::string_view, std::string_view>& p) const { return CombineHashes(std::hash<std::string_view>()(p.first), std::hash<std::string_view>()(p.second)); } }; + + std::unordered_map<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>, TPairHash> expects; + const auto t = TInstant::Now(); + for (auto& sample : TpchSamples) { + if (std::get<0U>(sample) <= border) { + const auto& ins = expects.emplace(std::pair<std::string_view, std::string_view>{std::get<1U>(sample), std::get<2U>(sample)}, std::pair<ui64, std::array<double, 5U>>{0ULL, {0., 0., 0., 0., 0.}}); + auto& item = ins.first->second; + ++item.first; + std::get<0U>(item.second) += std::get<3U>(sample); + std::get<1U>(item.second) += std::get<5U>(sample); + std::get<2U>(item.second) += std::get<6U>(sample); + const auto v = std::get<3U>(sample) * (1. - std::get<5U>(sample)); + std::get<3U>(item.second) += v; + std::get<4U>(item.second) += v * (1. + std::get<4U>(sample)); + } + } + for (auto& item : expects) { + std::get<1U>(item.second.second) /= item.second.first; + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<std::pair<std::string, std::string>, std::pair<ui64, std::array<double, 5U>>>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> l, const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui64>::Id), + pb.NewDataType(NUdf::TDataType<const char*>::Id), + pb.NewDataType(NUdf::TDataType<const char*>::Id), + pb.NewDataType(NUdf::TDataType<double>::Id), + pb.NewDataType(NUdf::TDataType<double>::Id), + pb.NewDataType(NUdf::TDataType<double>::Id), + pb.NewDataType(NUdf::TDataType<double>::Id) + })); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.CombineCore( + pb.Map(pb.Filter(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) { return pb.AggrLessOrEqual(pb.Nth(item, 0U), pb.NewDataLiteral<ui64>(border)); } + ), + [&](TRuntimeNode item) { return pb.NewTuple({pb.Nth(item, 1U), pb.Nth(item, 2U),pb.Nth(item, 3U),pb.Nth(item, 4U),pb.Nth(item, 5U),pb.Nth(item, 6U)}); } ), + [&](TRuntimeNode item) { return pb.NewTuple({pb.Nth(item, 0U), pb.Nth(item, 1U)}); }, + [&](TRuntimeNode, TRuntimeNode item) { + const auto price = pb.Nth(item, 2U); + const auto disco = pb.Nth(item, 4U); + const auto v = pb.Mul(price, pb.Sub(pb.NewDataLiteral<double>(1.), disco)); + return pb.NewTuple({pb.NewDataLiteral<ui64>(1ULL), price, disco, pb.Nth(item, 5U), v, pb.Mul(v, pb.Add(pb.NewDataLiteral<double>(1.), pb.Nth(item, 3U))) }); + }, + [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { + const auto price = pb.Nth(item, 2U); + const auto disco = pb.Nth(item, 4U); + const auto v = pb.Mul(price, pb.Sub(pb.NewDataLiteral<double>(1.), disco)); + return pb.NewTuple({pb.Increment(pb.Nth(state, 0U)), pb.AggrAdd(pb.Nth(state, 1U), price), pb.AggrAdd(pb.Nth(state, 2U), disco), pb.AggrAdd(pb.Nth(state, 3U), pb.Nth(item, 5U)), pb.AggrAdd(pb.Nth(state, 4U), v), pb.AggrAdd(pb.Nth(state, 5U), pb.Mul(v, pb.Add(pb.NewDataLiteral<double>(1.), pb.Nth(item, 3U)))) }); + }, + [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({pb.Nth(key, 0U), pb.Nth(key, 1U), pb.Nth(state, 0U), pb.Nth(state, 1U), pb.Div(pb.Nth(state, 2U), pb.Nth(state, 0U)), pb.Nth(state, 3U), pb.Nth(state, 4U), pb.Nth(state, 5U)})); }, + 0ULL + )); + + const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(TpchSamples.size(), items)); + for (const auto& sample : TpchSamples) { + NUdf::TUnboxedValue* elements = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(7U, elements); + elements[0] = NUdf::TUnboxedValuePod(std::get<0U>(sample)); + elements[1] = NUdf::TUnboxedValuePod::Embedded(std::get<1U>(sample)); + elements[2] = NUdf::TUnboxedValuePod::Embedded(std::get<2U>(sample)); + elements[3] = NUdf::TUnboxedValuePod(std::get<3U>(sample)); + elements[4] = NUdf::TUnboxedValuePod(std::get<4U>(sample)); + elements[5] = NUdf::TUnboxedValuePod(std::get<5U>(sample)); + elements[6] = NUdf::TUnboxedValuePod(std::get<6U>(sample)); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + const auto elements = ptr[i].GetElements(); + two.emplace_back(std::make_pair(elements[0].AsStringRef(), elements[1].AsStringRef()), std::pair<ui64, std::array<double, 5U>>{elements[2].template Get<ui64>(), {elements[3].template Get<double>(), elements[4].template Get<double>(), elements[5].template Get<double>(), elements[6].template Get<double>(), elements[7].template Get<double>()}}); + } + + std::sort(two.begin(), two.end(), [](const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> l, const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } +} +#endif +} // NMiniKQL +} // NKikimr diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_compare_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_compare_ut.cpp new file mode 100644 index 00000000000..05f1e5bd882 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_compare_ut.cpp @@ -0,0 +1,1119 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/mkql_node_printer.h> +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> + +#include <ydb/library/yql/minikql/computation/mkql_computation_node_impl.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_list_adapter.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> + +#include <cfloat> +#include <utility> + +namespace NKikimr { +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLCompareTest) { + Y_UNIT_TEST_LLVM(SqlString) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto data0 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id); + auto data1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("010")); + auto data2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("020")); + + auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id); + auto optionalType = pb.NewOptionalType(dataType); + auto pairType = pb.NewTupleType({optionalType, optionalType}); + auto list = pb.NewList(pairType, { + pb.NewTuple({data0, data0}), + pb.NewTuple({data0, data1}), + pb.NewTuple({data0, data2}), + + pb.NewTuple({data1, data0}), + pb.NewTuple({data1, data1}), + pb.NewTuple({data1, data2}), + + pb.NewTuple({data2, data0}), + pb.NewTuple({data2, data1}), + pb.NewTuple({data2, data2}) + }); + + auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.NewTuple({ + pb.Equals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.NotEquals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.Less(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.LessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.Greater(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.GreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)) + }); + }); + + auto graph = setup.BuildGraph(pgmReturn); + auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); // == + UNIT_ASSERT(!item.GetElement(1)); // != + UNIT_ASSERT(!item.GetElement(2)); // < + UNIT_ASSERT(!item.GetElement(3)); // <= + UNIT_ASSERT(!item.GetElement(4)); // > + UNIT_ASSERT(!item.GetElement(5)); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); // == + UNIT_ASSERT(!item.GetElement(1)); // != + UNIT_ASSERT(!item.GetElement(2)); // < + UNIT_ASSERT(!item.GetElement(3)); // <= + UNIT_ASSERT(!item.GetElement(4)); // > + UNIT_ASSERT(!item.GetElement(5)); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); // == + UNIT_ASSERT(!item.GetElement(1)); // != + UNIT_ASSERT(!item.GetElement(2)); // < + UNIT_ASSERT(!item.GetElement(3)); // <= + UNIT_ASSERT(!item.GetElement(4)); // > + UNIT_ASSERT(!item.GetElement(5)); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); // == + UNIT_ASSERT(!item.GetElement(1)); // != + UNIT_ASSERT(!item.GetElement(2)); // < + UNIT_ASSERT(!item.GetElement(3)); // <= + UNIT_ASSERT(!item.GetElement(4)); // > + UNIT_ASSERT(!item.GetElement(5)); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); // == + UNIT_ASSERT(!item.GetElement(1)); // != + UNIT_ASSERT(!item.GetElement(2)); // < + UNIT_ASSERT(!item.GetElement(3)); // <= + UNIT_ASSERT(!item.GetElement(4)); // > + UNIT_ASSERT(!item.GetElement(5)); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(AggrString) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto data0 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id); + auto data1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("010")); + auto data2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("020")); + + auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id); + auto optionalType = pb.NewOptionalType(dataType); + auto pairType = pb.NewTupleType({optionalType, optionalType}); + auto list = pb.NewList(pairType, { + pb.NewTuple({data0, data0}), + pb.NewTuple({data0, data1}), + pb.NewTuple({data0, data2}), + + pb.NewTuple({data1, data0}), + pb.NewTuple({data1, data1}), + pb.NewTuple({data1, data2}), + + pb.NewTuple({data2, data0}), + pb.NewTuple({data2, data1}), + pb.NewTuple({data2, data2}) + }); + + auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.NewTuple({ + pb.AggrEquals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.AggrNotEquals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.AggrLess(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.AggrLessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.AggrGreater(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.AggrGreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)) + }); + }); + + auto graph = setup.BuildGraph(pgmReturn); + auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(SqlFloats) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto data1 = pb.NewDataLiteral<float>(-7.0f); + auto data2 = pb.NewDataLiteral<float>(3.0f); + auto data3 = pb.NewDataLiteral<float>(0.0f*HUGE_VALF); + auto data4 = pb.NewDataLiteral<double>(-7.0); + auto data5 = pb.NewDataLiteral<double>(3.0); + auto data6 = pb.NewDataLiteral<double>(0.0*HUGE_VAL); + + auto pairType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<float>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)}); + auto list = pb.NewList(pairType, { + pb.NewTuple({data1, data4}), + pb.NewTuple({data1, data5}), + pb.NewTuple({data1, data6}), + + pb.NewTuple({data2, data4}), + pb.NewTuple({data2, data5}), + pb.NewTuple({data2, data6}), + + pb.NewTuple({data3, data4}), + pb.NewTuple({data3, data5}), + pb.NewTuple({data3, data6}), + }); + + auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.NewTuple({ + pb.Equals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.NotEquals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.Less(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.LessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.Greater(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.GreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)) + }); + }); + + auto graph = setup.BuildGraph(pgmReturn); + auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(AggrFloats) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto data1 = pb.NewDataLiteral<float>(-7.0f); + auto data2 = pb.NewDataLiteral<float>(3.0f); + auto data3 = pb.NewDataLiteral<float>(0.0f*HUGE_VALF); + + auto pairType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<float>::Id), pb.NewDataType(NUdf::TDataType<float>::Id)}); + auto list = pb.NewList(pairType, { + pb.NewTuple({data1, data1}), + pb.NewTuple({data1, data2}), + pb.NewTuple({data1, data3}), + + pb.NewTuple({data2, data1}), + pb.NewTuple({data2, data2}), + pb.NewTuple({data2, data3}), + + pb.NewTuple({data3, data1}), + pb.NewTuple({data3, data2}), + pb.NewTuple({data3, data3}), + }); + + auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.NewTuple({ + pb.AggrEquals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.AggrNotEquals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.AggrLess(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.AggrLessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.AggrGreater(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.AggrGreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)) + }); + }); + + auto graph = setup.BuildGraph(pgmReturn); + auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(SqlSigned) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto data1 = pb.NewDataLiteral<i16>(-1); + auto data2 = pb.NewDataLiteral<i16>(+1); + auto data3 = pb.NewDataLiteral<i32>(-1); + auto data4 = pb.NewDataLiteral<i32>(+1); + + auto pairType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i16>::Id), pb.NewDataType(NUdf::TDataType<i32>::Id)}); + auto list = pb.NewList(pairType, { + pb.NewTuple({data1, data3}), + pb.NewTuple({data1, data4}), + + pb.NewTuple({data2, data3}), + pb.NewTuple({data2, data4}) + }); + + auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.NewTuple({ + pb.Equals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.NotEquals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.Less(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.LessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.Greater(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.GreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)) + }); + }); + + auto graph = setup.BuildGraph(pgmReturn); + auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(SqlSignedAndUnsigned) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto data1 = pb.NewDataLiteral<i8>(-1); + auto data2 = pb.NewDataLiteral<i8>(127); + auto data3 = pb.NewDataLiteral<ui8>(0xFF); + auto data4 = pb.NewDataLiteral<ui8>(127); + + auto pairType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i8>::Id), pb.NewDataType(NUdf::TDataType<ui8>::Id)}); + auto list = pb.NewList(pairType, { + pb.NewTuple({data1, data3}), + pb.NewTuple({data1, data4}), + + pb.NewTuple({data2, data3}), + pb.NewTuple({data2, data4}) + }); + + auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.NewTuple({ + pb.Equals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.NotEquals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.Less(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.LessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.Greater(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.GreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)) + }); + }); + + auto graph = setup.BuildGraph(pgmReturn); + auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(SqlUnsignedAndSigned) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto data1 = pb.NewDataLiteral<ui16>(0); + auto data2 = pb.NewDataLiteral<ui16>(0xFFFF); + auto data3 = pb.NewDataLiteral<i64>(-1); + auto data4 = pb.NewDataLiteral<i64>(0xFFFF); + + auto pairType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui16>::Id), pb.NewDataType(NUdf::TDataType<i64>::Id)}); + auto list = pb.NewList(pairType, { + pb.NewTuple({data1, data3}), + pb.NewTuple({data1, data4}), + + pb.NewTuple({data2, data3}), + pb.NewTuple({data2, data4}) + }); + + auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.NewTuple({ + pb.Equals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.NotEquals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.Less(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.LessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.Greater(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.GreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)) + }); + }); + + auto graph = setup.BuildGraph(pgmReturn); + auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(SimpleSqlString) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto opt1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1)); + auto opt2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2)); + auto optEmpty = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); + auto pgmReturn = pb.NewEmptyList(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<bool>::Id))); + + pgmReturn = pb.Append(pgmReturn, pb.Equals(opt1, opt1)); + pgmReturn = pb.Append(pgmReturn, pb.Equals(opt1, opt2)); + pgmReturn = pb.Append(pgmReturn, pb.Equals(opt1, optEmpty)); + pgmReturn = pb.Append(pgmReturn, pb.Equals(optEmpty, opt2)); + pgmReturn = pb.Append(pgmReturn, pb.Equals(optEmpty, optEmpty)); + + pgmReturn = pb.Append(pgmReturn, pb.NotEquals(opt1, opt1)); + pgmReturn = pb.Append(pgmReturn, pb.NotEquals(opt1, opt2)); + pgmReturn = pb.Append(pgmReturn, pb.NotEquals(opt1, optEmpty)); + pgmReturn = pb.Append(pgmReturn, pb.NotEquals(optEmpty, opt2)); + pgmReturn = pb.Append(pgmReturn, pb.NotEquals(optEmpty, optEmpty)); + + pgmReturn = pb.Append(pgmReturn, pb.Less(opt1, opt1)); + pgmReturn = pb.Append(pgmReturn, pb.Less(opt1, opt2)); + pgmReturn = pb.Append(pgmReturn, pb.Less(opt2, opt1)); + pgmReturn = pb.Append(pgmReturn, pb.Less(opt1, optEmpty)); + pgmReturn = pb.Append(pgmReturn, pb.Less(optEmpty, opt2)); + pgmReturn = pb.Append(pgmReturn, pb.Less(optEmpty, optEmpty)); + + pgmReturn = pb.Append(pgmReturn, pb.LessOrEqual(opt1, opt1)); + pgmReturn = pb.Append(pgmReturn, pb.LessOrEqual(opt1, opt2)); + pgmReturn = pb.Append(pgmReturn, pb.LessOrEqual(opt2, opt1)); + pgmReturn = pb.Append(pgmReturn, pb.LessOrEqual(opt1, optEmpty)); + pgmReturn = pb.Append(pgmReturn, pb.LessOrEqual(optEmpty, opt2)); + pgmReturn = pb.Append(pgmReturn, pb.LessOrEqual(optEmpty, optEmpty)); + + pgmReturn = pb.Append(pgmReturn, pb.Greater(opt1, opt1)); + pgmReturn = pb.Append(pgmReturn, pb.Greater(opt1, opt2)); + pgmReturn = pb.Append(pgmReturn, pb.Greater(opt2, opt1)); + pgmReturn = pb.Append(pgmReturn, pb.Greater(opt1, optEmpty)); + pgmReturn = pb.Append(pgmReturn, pb.Greater(optEmpty, opt2)); + pgmReturn = pb.Append(pgmReturn, pb.Greater(optEmpty, optEmpty)); + + pgmReturn = pb.Append(pgmReturn, pb.GreaterOrEqual(opt1, opt1)); + pgmReturn = pb.Append(pgmReturn, pb.GreaterOrEqual(opt1, opt2)); + pgmReturn = pb.Append(pgmReturn, pb.GreaterOrEqual(opt2, opt1)); + pgmReturn = pb.Append(pgmReturn, pb.GreaterOrEqual(opt1, optEmpty)); + pgmReturn = pb.Append(pgmReturn, pb.GreaterOrEqual(optEmpty, opt2)); + pgmReturn = pb.Append(pgmReturn, pb.GreaterOrEqual(optEmpty, optEmpty)); + + auto opt1s = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("A")); + auto opt2s = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("B")); + auto optEmptys = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id); + + pgmReturn = pb.Append(pgmReturn, pb.Equals(opt1s, opt1s)); + pgmReturn = pb.Append(pgmReturn, pb.Equals(opt1s, opt2s)); + pgmReturn = pb.Append(pgmReturn, pb.Equals(opt1s, optEmptys)); + pgmReturn = pb.Append(pgmReturn, pb.Equals(optEmptys, opt2s)); + pgmReturn = pb.Append(pgmReturn, pb.Equals(optEmptys, optEmptys)); + + pgmReturn = pb.Append(pgmReturn, pb.NotEquals(opt1s, opt1s)); + pgmReturn = pb.Append(pgmReturn, pb.NotEquals(opt1s, opt2s)); + pgmReturn = pb.Append(pgmReturn, pb.NotEquals(opt1s, optEmptys)); + pgmReturn = pb.Append(pgmReturn, pb.NotEquals(optEmptys, opt2s)); + pgmReturn = pb.Append(pgmReturn, pb.NotEquals(optEmptys, optEmptys)); + + auto graph = setup.BuildGraph(pgmReturn); + auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + // equals + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // not equals + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // less + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // less or equal + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // greater + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // greater or equal + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // equals - string + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // not equals - string + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TzMin) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui16>(1ULL); + const auto data2 = pb.NewDataLiteral<ui16>(7ULL); + + const auto data3 = pb.NewDataLiteral<ui32>(1ULL); + const auto data4 = pb.NewDataLiteral<ui32>(7ULL); + + const auto zones = pb.ListFromRange(data1, data2, data1); + const auto dates = pb.ListFromRange(data3, data4, data3); + + const auto source = pb.Map(pb.Zip({pb.Reverse(dates), zones}), + [&](TRuntimeNode item) { + return pb.AddTimezone(pb.ToIntegral(pb.Nth(item, 0U), pb.NewDataType(NUdf::EDataSlot::Datetime, true)), pb.Nth(item, 1U)); + }); + + const auto pgmReturn = pb.ToString(pb.Unwrap(pb.Fold1(source, + [&](TRuntimeNode item) { return item; }, + [&](TRuntimeNode item, TRuntimeNode state) { return pb.Min(item, state); } + ), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), __FILE__, __LINE__, 0)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto result = graph->GetValue(); + UNBOXED_VALUE_STR_EQUAL(result, "1970-01-01T03:00:01,Africa/Asmara"); + } + + Y_UNIT_TEST_LLVM(TzMax) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui16>(1ULL); + const auto data2 = pb.NewDataLiteral<ui16>(7ULL); + + const auto data3 = pb.NewDataLiteral<ui32>(1ULL); + const auto data4 = pb.NewDataLiteral<ui32>(7ULL); + + const auto zones = pb.ListFromRange(data1, data2, data1); + const auto dates = pb.ListFromRange(data3, data4, data3); + + const auto source = pb.Map(pb.Zip({dates, pb.Reverse(zones)}), + [&](TRuntimeNode item) { + return pb.AddTimezone(pb.ToIntegral(pb.Nth(item, 0U), pb.NewDataType(NUdf::EDataSlot::Datetime, true)), pb.Nth(item, 1U)); + }); + + const auto pgmReturn = pb.ToString(pb.Unwrap(pb.Fold1(source, + [&](TRuntimeNode item) { return item; }, + [&](TRuntimeNode item, TRuntimeNode state) { return pb.Max(item, state); } + ), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), __FILE__, __LINE__, 0)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto result = graph->GetValue(); + UNBOXED_VALUE_STR_EQUAL(result, "1970-01-01T03:00:06,Europe/Moscow"); + } + + Y_UNIT_TEST_LLVM(TzAggrMin) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui16>(1ULL); + const auto data2 = pb.NewDataLiteral<ui16>(7ULL); + + const auto data3 = pb.NewDataLiteral<ui32>(1ULL); + const auto data4 = pb.NewDataLiteral<ui32>(7ULL); + + const auto zones = pb.ListFromRange(data1, data2, data1); + const auto dates = pb.ListFromRange(data3, data4, data3); + + const auto source = pb.FlatMap(zones, + [&](TRuntimeNode zone) { + return pb.Map(dates, + [&](TRuntimeNode date) { + return pb.AddTimezone(pb.ToIntegral(date, pb.NewDataType(NUdf::EDataSlot::Datetime, true)), zone); + }); + }); + + const auto pgmReturn = pb.ToString(pb.Unwrap(pb.Fold1(source, + [&](TRuntimeNode item) { return item; }, + [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrMin(item, state); } + ), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), __FILE__, __LINE__, 0)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto result = graph->GetValue(); + UNBOXED_VALUE_STR_EQUAL(result, "1970-01-01T03:00:01,Europe/Moscow"); + } + + Y_UNIT_TEST_LLVM(TzAggrMax) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui16>(1ULL); + const auto data2 = pb.NewDataLiteral<ui16>(7ULL); + + const auto data3 = pb.NewDataLiteral<ui32>(1ULL); + const auto data4 = pb.NewDataLiteral<ui32>(7ULL); + + const auto zones = pb.ListFromRange(data1, data2, data1); + const auto dates = pb.ListFromRange(data3, data4, data3); + + const auto source = pb.FlatMap(dates, + [&](TRuntimeNode date) { + return pb.Map(zones, + [&](TRuntimeNode zone) { + return pb.AddTimezone(pb.ToIntegral(date, pb.NewDataType(NUdf::EDataSlot::Datetime, true)), zone); + }); + }); + + const auto pgmReturn = pb.ToString(pb.Unwrap(pb.Fold1(source, + [&](TRuntimeNode item) { return item; }, + [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrMax(item, state); } + ), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), __FILE__, __LINE__, 0)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto result = graph->GetValue(); + UNBOXED_VALUE_STR_EQUAL(result, "1970-01-01T03:00:06,Africa/Asmara"); + } + + Y_UNIT_TEST_LLVM(TestAggrMinMaxFloats) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<float>(0.0f*HUGE_VALF); + const auto data2 = pb.NewDataLiteral<float>(HUGE_VALF); + const auto data3 = pb.NewDataLiteral<float>(3.14f); + const auto data4 = pb.NewDataLiteral<float>(-2.13f); + const auto data5 = pb.NewDataLiteral<float>(-HUGE_VALF); + const auto dataType = pb.NewDataType(NUdf::TDataType<float>::Id); + const auto list = pb.NewList(dataType, {data1, data2, data3, data4, data5}); + const auto pgmReturn = pb.FlatMap(list, + [&](TRuntimeNode left) { + return pb.Map(list, + [&](TRuntimeNode right) { + return pb.NewTuple({pb.AggrMin(left, right), pb.AggrMax(left, right)}); + }); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(std::isnan(item.GetElement(0).Get<float>())); + UNIT_ASSERT(std::isnan(item.GetElement(1).Get<float>())); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), HUGE_VALF); + UNIT_ASSERT(std::isnan(item.GetElement(1).Get<float>())); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), 3.14f); + UNIT_ASSERT(std::isnan(item.GetElement(1).Get<float>())); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -2.13f); + UNIT_ASSERT(std::isnan(item.GetElement(1).Get<float>())); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -HUGE_VALF); + UNIT_ASSERT(std::isnan(item.GetElement(1).Get<float>())); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), HUGE_VALF); + UNIT_ASSERT(std::isnan(item.GetElement(1).Get<float>())); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), HUGE_VALF); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), 3.14f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), HUGE_VALF); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -2.13f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), HUGE_VALF); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), HUGE_VALF); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), 3.14f); + UNIT_ASSERT(std::isnan(item.GetElement(1).Get<float>())); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), 3.14f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), HUGE_VALF); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), 3.14f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), 3.14f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -2.13f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), 3.14f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), 3.14f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -2.13f); + UNIT_ASSERT(std::isnan(item.GetElement(1).Get<float>())); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -2.13f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), HUGE_VALF); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -2.13f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), 3.14f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -2.13f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), -2.13f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), -2.13f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -HUGE_VALF); + UNIT_ASSERT(std::isnan(item.GetElement(1).Get<float>())); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), HUGE_VALF); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), 3.14f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), -2.13f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), -HUGE_VALF); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp new file mode 100644 index 00000000000..eefce3227a2 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp @@ -0,0 +1,4799 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/mkql_node_printer.h> +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> + +#include <ydb/library/yql/minikql/computation/mkql_computation_node_impl.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> + +#include <cfloat> +#include <utility> +#include <random> + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + +constexpr auto TotalSambles = +#ifndef NDEBUG +333333U; +#else +33333333ULL; +#endif + +} + +std::vector<std::pair<i8, double>> MakeSamples() { + std::default_random_engine eng; + std::uniform_int_distribution<i8> keys(-100, +100); + std::uniform_real_distribution<double> unif(-999.0, +999.0); + + std::vector<std::pair<i8, double>> samples(TotalSambles); + + eng.seed(std::time(nullptr)); + std::generate(samples.begin(), samples.end(), std::bind(&std::make_pair<i8, double>, std::bind(std::move(keys), std::move(eng)), std::bind(std::move(unif), std::move(eng)))); + return samples; +} + +std::vector<std::pair<ui16, double>> MakeOtherSamples() { + std::default_random_engine eng; + std::uniform_int_distribution<ui16> keys(0U, 65535U); + std::uniform_real_distribution<double> unif(-999.0, +999.0); + + std::vector<std::pair<ui16, double>> samples(TotalSambles); + + eng.seed(std::time(nullptr)); + std::generate(samples.begin(), samples.end(), std::bind(&std::make_pair<ui16, double>, std::bind(std::move(keys), std::move(eng)), std::bind(std::move(unif), std::move(eng)))); + return samples; +} + +std::vector<std::tuple<ui64, std::string, std::string, double, double, double, double>> MakeTpchSamples() { + std::default_random_engine eng; + std::uniform_int_distribution<ui64> dates(694303200000000ULL, 9124596000000005ULL); + std::uniform_int_distribution<ui8> keys(0U, 3U); + std::uniform_real_distribution<double> prices(900., 105000.0); + std::uniform_real_distribution<double> taxes(0., 0.08); + std::uniform_real_distribution<double> discs(0., 0.1); + std::uniform_real_distribution<double> qntts(1., 50.); + + std::random_device rd; + std::mt19937 gen(rd()); + + std::vector<std::tuple<ui64, std::string, std::string, double, double, double, double>> samples(TotalSambles); + + eng.seed(std::time(nullptr)); + + std::generate(samples.begin(), samples.end(), [&]() { + switch(keys(gen)) { + case 0U: return std::make_tuple(dates(gen), "N", "O", prices(gen), taxes(gen), discs(gen), qntts(gen)); + case 1U: return std::make_tuple(dates(gen), "A", "F", prices(gen), taxes(gen), discs(gen), qntts(gen)); + case 2U: return std::make_tuple(dates(gen), "N", "F", prices(gen), taxes(gen), discs(gen), qntts(gen)); + case 3U: return std::make_tuple(dates(gen), "R", "F", prices(gen), taxes(gen), discs(gen), qntts(gen)); + } + Y_FAIL("Unexpected"); + }); + return samples; +} + +extern const std::vector<std::pair<i8, double>> I8Samples = MakeSamples(); +extern const std::vector<std::pair<ui16, double>> Ui16Samples = MakeOtherSamples(); +extern const std::vector<std::tuple<ui64, std::string, std::string, double, double, double, double>> TpchSamples = MakeTpchSamples(); + +Y_UNIT_TEST_SUITE(TMiniKQLComputationNodeTest) { + Y_UNIT_TEST_LLVM(TestEmbeddedByteAt) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<ui32>(0); + const auto data1 = pb.NewDataLiteral<ui32>(3); + const auto data2 = pb.NewDataLiteral<ui32>(9); + const auto data3 = pb.NewDataLiteral<ui32>(13); + const auto data4 = pb.NewDataLiteral<ui32>(Max<ui32>()); + + const auto str = pb.NewDataLiteral<NUdf::EDataSlot::String>("0123456789AB"); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.ByteAt(str, item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 0x30); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 0x33); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 0x39); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestStringByteAt) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<ui32>(0); + const auto data1 = pb.NewDataLiteral<ui32>(3); + const auto data2 = pb.NewDataLiteral<ui32>(21); + const auto data3 = pb.NewDataLiteral<ui32>(22); + const auto data4 = pb.NewDataLiteral<ui32>(Max<ui32>()); + + const auto str = pb.NewDataLiteral<NUdf::EDataSlot::String>("0123456789ABCDEFGHIJKL"); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.ByteAt(str, item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 0x30); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 0x33); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 0x4C); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestCountBitsUI8) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<ui8>(0x00); + const auto data1 = pb.NewDataLiteral<ui8>(0x01); + const auto data2 = pb.NewDataLiteral<ui8>(0x10); + const auto data3 = pb.NewDataLiteral<ui8>(0x13); + const auto data4 = pb.NewDataLiteral<ui8>(0xF0); + const auto data5 = pb.NewDataLiteral<ui8>(0x0F); + const auto data6 = pb.NewDataLiteral<ui8>(0x55); + const auto data7 = pb.NewDataLiteral<ui8>(0xAA); + const auto data8 = pb.NewDataLiteral<ui8>(0x7F); + const auto data9 = pb.NewDataLiteral<ui8>(0xFF); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui8>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.CountBits(item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 7); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 8); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestCountBitsUI16) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<ui16>(0x0000); + const auto data1 = pb.NewDataLiteral<ui16>(0x0100); + const auto data2 = pb.NewDataLiteral<ui16>(0x0010); + const auto data3 = pb.NewDataLiteral<ui16>(0x0130); + const auto data4 = pb.NewDataLiteral<ui16>(0xF000); + const auto data5 = pb.NewDataLiteral<ui16>(0x0F00); + const auto data6 = pb.NewDataLiteral<ui16>(0x0550); + const auto data7 = pb.NewDataLiteral<ui16>(0xA00A); + const auto data8 = pb.NewDataLiteral<ui16>(0x700F); + const auto data9 = pb.NewDataLiteral<ui16>(0x0FF0); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.CountBits(item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 7); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 8); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestShiftLeft) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<ui8>(0); + const auto data1 = pb.NewDataLiteral<ui8>(1); + const auto data2 = pb.NewDataLiteral<ui8>(7); + const auto data3 = pb.NewDataLiteral<ui8>(10); + const auto data4 = pb.NewDataLiteral<ui8>(30); + const auto data5 = pb.NewDataLiteral<ui8>(32); + const auto data6 = pb.NewDataLiteral<ui8>(40); + const auto data7 = pb.NewDataLiteral<ui8>(57); + const auto data8 = pb.NewDataLiteral<ui8>(200); + const auto data9 = pb.NewDataLiteral<ui8>(255); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui8>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.ShiftLeft(pb.NewDataLiteral<ui32>(0x830500F1U), item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x830500F1U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x060A01E2U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x82807880U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x1403C400U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x40000000U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestShiftRight) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<ui8>(0); + const auto data1 = pb.NewDataLiteral<ui8>(1); + const auto data2 = pb.NewDataLiteral<ui8>(7); + const auto data3 = pb.NewDataLiteral<ui8>(10); + const auto data4 = pb.NewDataLiteral<ui8>(30); + const auto data5 = pb.NewDataLiteral<ui8>(32); + const auto data6 = pb.NewDataLiteral<ui8>(40); + const auto data7 = pb.NewDataLiteral<ui8>(57); + const auto data8 = pb.NewDataLiteral<ui8>(200); + const auto data9 = pb.NewDataLiteral<ui8>(255); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui8>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.ShiftRight(pb.NewDataLiteral<ui32>(0x830500F1U), item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x830500F1U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x41828078U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x01060A01U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x20C140U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x2U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestRotLeft) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<ui8>(0); + const auto data1 = pb.NewDataLiteral<ui8>(1); + const auto data2 = pb.NewDataLiteral<ui8>(7); + const auto data3 = pb.NewDataLiteral<ui8>(10); + const auto data4 = pb.NewDataLiteral<ui8>(30); + const auto data5 = pb.NewDataLiteral<ui8>(32); + const auto data6 = pb.NewDataLiteral<ui8>(40); + const auto data7 = pb.NewDataLiteral<ui8>(57); + const auto data8 = pb.NewDataLiteral<ui8>(200); + const auto data9 = pb.NewDataLiteral<ui8>(255); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui8>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.RotLeft(pb.NewDataLiteral<ui16>(0x87F5U), item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0x87F5U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0x0FEBU); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xFAC3U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xD61FU); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0x61FDU); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0x87F5U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xF587U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xEB0FU); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xF587U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xC3FAU); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestRotRight) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<ui8>(0); + const auto data1 = pb.NewDataLiteral<ui8>(1); + const auto data2 = pb.NewDataLiteral<ui8>(7); + const auto data3 = pb.NewDataLiteral<ui8>(10); + const auto data4 = pb.NewDataLiteral<ui8>(30); + const auto data5 = pb.NewDataLiteral<ui8>(32); + const auto data6 = pb.NewDataLiteral<ui8>(40); + const auto data7 = pb.NewDataLiteral<ui8>(57); + const auto data8 = pb.NewDataLiteral<ui8>(200); + const auto data9 = pb.NewDataLiteral<ui8>(255); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui8>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.RotRight(pb.NewDataLiteral<ui16>(0x87F5U), item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0x87F5U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xC3FAU); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xEB0FU); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xFD61U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0x1FD6U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0x87F5U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xF587U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xFAC3U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xF587U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0x0FEBU); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestConvertIntToBool) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<i64>(0); + const auto data1 = pb.NewDataLiteral<i64>(1); + const auto data2 = pb.NewDataLiteral<i64>(2); + const auto data3 = pb.NewDataLiteral<i64>(1024); + const auto data4 = pb.NewDataLiteral<i64>(-1); + const auto dataType = pb.NewDataType(NUdf::TDataType<i64>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Convert(item, pb.NewDataType(NUdf::TDataType<bool>::Id)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.template Get<bool>()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.template Get<bool>()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.template Get<bool>()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.template Get<bool>()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.template Get<bool>()); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFloatAbs) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<float>::Id); + const auto data1 = pb.NewDataLiteral<float>(11.433f); + const auto data2 = pb.NewDataLiteral<float>(-3.14f); + const auto data3 = pb.NewDataLiteral<float>(0.0f); + const auto data4 = pb.NewDataLiteral<float>(-HUGE_VALF); + const auto list = pb.NewList(dataType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Abs(item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 11.433f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 3.14f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 0.0f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), HUGE_VALF); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestIntegerAbs) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id); + const auto data1 = pb.NewDataLiteral<i32>(1334); + const auto data2 = pb.NewDataLiteral<i32>(-4378); + const auto data3 = pb.NewDataLiteral<i32>(0); + const auto data4 = pb.NewDataLiteral<i32>(std::numeric_limits<i32>::min()); + const auto list = pb.NewList(dataType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Abs(item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 1334); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 4378); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), std::numeric_limits<i32>::min()); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestToIntegral) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto ten = pb.ToIntegral(pb.NewDataLiteral(float(10.0)), pb.NewDataType(NUdf::TDataType<i32>::Id, true)); + const auto two = pb.ToIntegral(pb.NewDataLiteral(float(2.0)), pb.NewDataType(NUdf::TDataType<ui32>::Id, true)); + const auto pgmReturn = pb.NewTuple({ten, two}); + + const auto graph = setup.BuildGraph(pgmReturn); + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetElement(0).template Get<i32>(), 10); + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetElement(1).template Get<ui32>(), 2U); + } + + Y_UNIT_TEST_LLVM(TestFloatToI16) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral(0.0f); + const auto data1 = pb.NewDataLiteral(0.0f*HUGE_VALF); + const auto data2 = pb.NewDataLiteral(-3.14f); + const auto data3 = pb.NewDataLiteral(12121324.0f); + const auto data4 = pb.NewDataLiteral(-7898.8f); + const auto data5 = pb.NewDataLiteral(210000.0f); + const auto data6 = pb.NewDataLiteral(HUGE_VALF); + const auto data7 = pb.NewDataLiteral(-HUGE_VALF); + const auto data8 = pb.NewDataLiteral(-HUGE_VALF); + const auto data9 = pb.NewDataLiteral(FLT_MIN/2.0f); + const auto type = pb.NewDataType(NUdf::TDataType<float>::Id); + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i16>::Id, true)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), -3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), -7898); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), 0); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestDoubleToBool) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral(0.0); + const auto data1 = pb.NewDataLiteral(0.0*HUGE_VAL); + const auto data2 = pb.NewDataLiteral(-3.14); + const auto data3 = pb.NewDataLiteral(12121324.0); + const auto data4 = pb.NewDataLiteral(-7898.8); + const auto data5 = pb.NewDataLiteral(210000.0); + const auto data6 = pb.NewDataLiteral(HUGE_VAL); + const auto data7 = pb.NewDataLiteral(-HUGE_VAL); + const auto data8 = pb.NewDataLiteral(-HUGE_VAL); + const auto data9 = pb.NewDataLiteral(DBL_MIN/2.0); + const auto type = pb.NewDataType(NUdf::TDataType<double>::Id); + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<bool>::Id, true)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestDoubleToUI32) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral(0.0); + const auto data1 = pb.NewDataLiteral(0.0*HUGE_VAL); + const auto data2 = pb.NewDataLiteral(-3.14); + const auto data3 = pb.NewDataLiteral(12121324.0); + const auto data4 = pb.NewDataLiteral(-7898.8); + const auto data5 = pb.NewDataLiteral(210000.0); + const auto data6 = pb.NewDataLiteral(HUGE_VAL); + const auto data7 = pb.NewDataLiteral(-HUGE_VAL); + const auto data8 = pb.NewDataLiteral(-HUGE_VAL); + const auto data9 = pb.NewDataLiteral(DBL_MIN/2.0); + const auto type = pb.NewDataType(NUdf::TDataType<double>::Id); + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui32>::Id, true)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 12121324U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 210000U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestUI64ToIntegral) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto type = pb.NewDataType(NUdf::TDataType<ui64>::Id); + const auto data0 = pb.NewDataLiteral<ui64>(0U); + const auto data1 = pb.NewDataLiteral<ui64>(1U); + const auto data2 = pb.NewDataLiteral<ui64>(std::numeric_limits<i8>::max()); + const auto data3 = pb.NewDataLiteral<ui64>(std::numeric_limits<ui8>::max()); + const auto data4 = pb.NewDataLiteral<ui64>(std::numeric_limits<i16>::max()); + const auto data5 = pb.NewDataLiteral<ui64>(std::numeric_limits<ui16>::max()); + const auto data6 = pb.NewDataLiteral<ui64>(std::numeric_limits<i32>::max()); + const auto data7 = pb.NewDataLiteral<ui64>(std::numeric_limits<ui32>::max()); + const auto data8 = pb.NewDataLiteral<ui64>(std::numeric_limits<i64>::max()); + const auto data9 = pb.NewDataLiteral<ui64>(std::numeric_limits<ui64>::max()); + + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.NewTuple({ + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i8>::Id, true)), + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui8>::Id, true)), + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i16>::Id, true)), + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui16>::Id, true)), + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i32>::Id, true)), + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui32>::Id, true)), + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i64>::Id, true)), + }); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[1].Get<ui8>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 0); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[1].Get<ui8>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 1); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), std::numeric_limits<i8>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[1].Get<ui8>(), std::numeric_limits<i8>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), std::numeric_limits<i8>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), std::numeric_limits<i8>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<i8>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), std::numeric_limits<i8>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), std::numeric_limits<i8>::max()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[1].Get<ui8>(), std::numeric_limits<ui8>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), std::numeric_limits<ui8>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), std::numeric_limits<ui8>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<ui8>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), std::numeric_limits<ui8>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), std::numeric_limits<ui8>::max()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), std::numeric_limits<i16>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), std::numeric_limits<i16>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<i16>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), std::numeric_limits<i16>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), std::numeric_limits<i16>::max()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT(!item.GetElements()[2]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), std::numeric_limits<ui16>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<ui16>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), std::numeric_limits<ui16>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), std::numeric_limits<ui16>::max()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT(!item.GetElements()[2]); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<i32>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), std::numeric_limits<i32>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), std::numeric_limits<i32>::max()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT(!item.GetElements()[2]); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT(!item.GetElements()[4]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), std::numeric_limits<ui32>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), std::numeric_limits<ui32>::max()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT(!item.GetElements()[2]); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT(!item.GetElements()[4]); + UNIT_ASSERT(!item.GetElements()[5]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), std::numeric_limits<i64>::max()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT(!item.GetElements()[2]); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT(!item.GetElements()[4]); + UNIT_ASSERT(!item.GetElements()[5]); + UNIT_ASSERT(!item.GetElements()[6]); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestI64ToIntegral) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto type = pb.NewDataType(NUdf::TDataType<i64>::Id); + const auto data0 = pb.NewDataLiteral<i64>(0); + const auto data1 = pb.NewDataLiteral<i64>(-1); + const auto data2 = pb.NewDataLiteral<i64>(std::numeric_limits<i8>::min()); + const auto data3 = pb.NewDataLiteral<i64>(std::numeric_limits<i8>::max()); + const auto data4 = pb.NewDataLiteral<i64>(std::numeric_limits<i16>::min()); + const auto data5 = pb.NewDataLiteral<i64>(std::numeric_limits<i16>::max()); + const auto data6 = pb.NewDataLiteral<i64>(std::numeric_limits<i32>::min()); + const auto data7 = pb.NewDataLiteral<i64>(std::numeric_limits<i32>::max()); + const auto data8 = pb.NewDataLiteral<i64>(std::numeric_limits<i64>::min()); + const auto data9 = pb.NewDataLiteral<i64>(std::numeric_limits<i64>::max()); + + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.NewTuple({ + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i8>::Id, true)), + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui8>::Id, true)), + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i16>::Id, true)), + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui16>::Id, true)), + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i32>::Id, true)), + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui32>::Id, true)), + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui64>::Id, true)), + }); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[1].Get<ui8>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<ui64>(), 0); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), -1); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), -1); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), -1); + UNIT_ASSERT(!item.GetElements()[5]); + UNIT_ASSERT(!item.GetElements()[6]); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), std::numeric_limits<i8>::min()); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), std::numeric_limits<i8>::min()); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<i8>::min()); + UNIT_ASSERT(!item.GetElements()[5]); + UNIT_ASSERT(!item.GetElements()[6]); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), std::numeric_limits<i8>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[1].Get<ui8>(), std::numeric_limits<i8>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), std::numeric_limits<i8>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), std::numeric_limits<i8>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<i8>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), std::numeric_limits<i8>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<ui64>(), std::numeric_limits<i8>::max()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), std::numeric_limits<i16>::min()); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<i16>::min()); + UNIT_ASSERT(!item.GetElements()[5]); + UNIT_ASSERT(!item.GetElements()[6]); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), std::numeric_limits<i16>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), std::numeric_limits<i16>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<i16>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), std::numeric_limits<i16>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<ui64>(), std::numeric_limits<i16>::max()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT(!item.GetElements()[2]); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<i32>::min()); + UNIT_ASSERT(!item.GetElements()[5]); + UNIT_ASSERT(!item.GetElements()[6]); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT(!item.GetElements()[2]); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<i32>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), std::numeric_limits<i32>::max()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<ui64>(), std::numeric_limits<i32>::max()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT(!item.GetElements()[2]); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT(!item.GetElements()[4]); + UNIT_ASSERT(!item.GetElements()[5]); + UNIT_ASSERT(!item.GetElements()[6]); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT(!item.GetElements()[2]); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT(!item.GetElements()[4]); + UNIT_ASSERT(!item.GetElements()[5]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<ui64>(), std::numeric_limits<i64>::max()); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFloatFromString) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("0.0"); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("NAN"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-3.14"); + const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("1212.00"); + const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-7898.8"); + const auto data5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("21E4"); + const auto data6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("+inf"); + const auto data7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-INF"); + const auto type = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.StrictFromString(item, pb.NewDataType(NUdf::TDataType<float>::Id)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 0.f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(std::isnan(item.template Get<float>())); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -3.14f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 1212.f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -7898.8f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 210000.f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), HUGE_VALF); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -HUGE_VALF); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestDoubleFromString) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("0.0"); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("NAN"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-3.14"); + const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("12121324.00"); + const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-7898.8"); + const auto data5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("21E4"); + const auto data6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("+inf"); + const auto data7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-INF"); + const auto type = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.StrictFromString(item, pb.NewDataType(NUdf::TDataType<double>::Id)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(std::isnan(item.template Get<double>())); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -3.14); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 12121324.0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -7898.8); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 210000.0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), HUGE_VAL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -HUGE_VAL); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFloatToString) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral(0.0f); + const auto data1 = pb.NewDataLiteral(0.0f*HUGE_VALF); + const auto data2 = pb.NewDataLiteral(-3.14f); + const auto data3 = pb.NewDataLiteral(1212.0f); + const auto data4 = pb.NewDataLiteral(-7898.8f); + const auto data5 = pb.NewDataLiteral(210000.0f); + const auto data6 = pb.NewDataLiteral(HUGE_VALF); + const auto data7 = pb.NewDataLiteral(-HUGE_VALF); + const auto type = pb.NewDataType(NUdf::TDataType<float>::Id); + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.ToString(item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "0"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "nan"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "-3.14"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "1212"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "-7898.8"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "210000"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "inf"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "-inf"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestInt64ToTimestamp) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral(i64(0LL)); + const auto data1 = pb.NewDataLiteral(i64(1LL)); + const auto data2 = pb.NewDataLiteral(i64(-1LL)); + const auto data3 = pb.NewDataLiteral(std::numeric_limits<i64>::min()); + const auto data4 = pb.NewDataLiteral(std::numeric_limits<i64>::max()); + const auto data5 = pb.NewDataLiteral(i64(4291747200000000LL)); + const auto data6 = pb.NewDataLiteral(i64(4291747199999999LL)); + const auto type = pb.NewDataType(NUdf::TDataType<i64>::Id); + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id, true)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 0ULL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 1ULL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 4291747199999999ULL); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFloatConvertToUint32) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral(0.0f); + const auto data3 = pb.NewDataLiteral(1212.0f); + const auto data5 = pb.NewDataLiteral(210000.0f); + const auto type = pb.NewDataType(NUdf::TDataType<float>::Id); + const auto list = pb.NewList(type, {data0, data3, data5}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Convert(item, pb.NewDataType(NUdf::TDataType<ui32>::Id)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1212U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 210000U); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFloatConvertToInt32) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral(0.0f); + const auto data2 = pb.NewDataLiteral(-3.14f); + const auto data3 = pb.NewDataLiteral(1212.0f); + const auto data4 = pb.NewDataLiteral(-7898.8f); + const auto data5 = pb.NewDataLiteral(210000.0f); + const auto type = pb.NewDataType(NUdf::TDataType<float>::Id); + const auto list = pb.NewList(type, {data0, data2, data3, data4, data5}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Convert(item, pb.NewDataType(NUdf::TDataType<i32>::Id)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 1212); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -7898); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 210000); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestDoubleConvertToInt64) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral(0.0); + const auto data2 = pb.NewDataLiteral(-3.14); + const auto data3 = pb.NewDataLiteral(1212.0); + const auto data4 = pb.NewDataLiteral(-7898.8); + const auto data5 = pb.NewDataLiteral(210000.0); + const auto type = pb.NewDataType(NUdf::TDataType<double>::Id); + const auto list = pb.NewList(type, {data0, data2, data3, data4, data5}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Convert(item, pb.NewDataType(NUdf::TDataType<i64>::Id)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), 0LL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), -3LL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), 1212LL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), -7898LL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), 210000LL); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestDoubleToInt64) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral(0.0); + const auto data2 = pb.NewDataLiteral(-3.14); + const auto data3 = pb.NewDataLiteral(1212.0); + const auto data4 = pb.NewDataLiteral(-7898.8); + const auto data5 = pb.NewDataLiteral(210000.0); + const auto data6 = pb.NewDataLiteral(9223372036854776000.0); + const auto type = pb.NewDataType(NUdf::TDataType<double>::Id); + const auto list = pb.NewList(type, {data0, data2, data3, data4, data5, data6}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.ToIntegral(item, pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i64>::Id))); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), 0LL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), -3LL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), 1212LL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), -7898LL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), 210000LL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestDoubleToString) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral(0.0); + const auto data1 = pb.NewDataLiteral(0.0*HUGE_VAL); + const auto data2 = pb.NewDataLiteral(-3.14); + const auto data3 = pb.NewDataLiteral(12121324.0); + const auto data4 = pb.NewDataLiteral(-7898.8); + const auto data5 = pb.NewDataLiteral(210000.0); + const auto data6 = pb.NewDataLiteral(HUGE_VAL); + const auto data7 = pb.NewDataLiteral(-HUGE_VAL); + const auto type = pb.NewDataType(NUdf::TDataType<double>::Id); + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.ToString(item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "0"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "nan"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "-3.14"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "12121324"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "-7898.8"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "210000"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "inf"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "-inf"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestNanvl) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewOptional(pb.NewDataLiteral(0.0f)); + const auto data1 = pb.NewOptional(pb.NewDataLiteral(0.0f*HUGE_VALF)); + const auto data2 = pb.NewOptional(pb.NewDataLiteral(HUGE_VALF)); + const auto data3 = pb.NewOptional(pb.NewDataLiteral(-HUGE_VALF)); + const auto data4 = pb.NewOptional(pb.NewDataLiteral(FLT_MIN/2.0f)); + const auto data5 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<float>::Id); + const auto data = pb.NewDataLiteral(3.14); + + const auto type = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<float>::Id)); + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Nanvl(item, data); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 3.14); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), HUGE_VAL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -HUGE_VAL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), FLT_MIN/2.0f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestConvertToFloat) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto seven = pb.NewDataLiteral(i32(7)); + const auto pgmReturn = pb.Convert(seven, pb.NewDataType(NUdf::TDataType<float>::Id)); + const auto graph = setup.BuildGraph(pgmReturn); + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().template Get<float>(), 7.0f); + } + + Y_UNIT_TEST_LLVM(TestAppend) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + auto pgmReturn = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id)); + pgmReturn = pb.Append(pgmReturn, pb.NewDataLiteral<ui32>(34)); + pgmReturn = pb.Append(pgmReturn, pb.NewDataLiteral<ui32>(56)); + + const auto graph = setup.BuildGraph(pgmReturn); + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 2); + + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 56); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestForkedAppend) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), {pb.NewDataLiteral<ui32>(34)}); + const auto list2 = pb.Append(list1, pb.NewDataLiteral<ui32>(56)); + const auto list3 = pb.Append(list1, pb.NewDataLiteral<ui32>(78)); + const auto pgmReturn = pb.NewTuple({list2, list3}); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator1 = graph->GetValue().GetElement(0).GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator1.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34); + UNIT_ASSERT(iterator1.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 56); + UNIT_ASSERT(!iterator1.Next(item)); + UNIT_ASSERT(!iterator1.Next(item)); + + const auto iterator2 = graph->GetValue().GetElement(1).GetListIterator(); + UNIT_ASSERT(iterator2.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34); + UNIT_ASSERT(iterator2.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 78); + UNIT_ASSERT(!iterator2.Next(item)); + UNIT_ASSERT(!iterator2.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestPrepend) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + auto pgmReturn = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id)); + pgmReturn = pb.Prepend(pb.NewDataLiteral<ui32>(34), pgmReturn); + pgmReturn = pb.Prepend(pb.NewDataLiteral<ui32>(56), pgmReturn); + + const auto graph = setup.BuildGraph(pgmReturn); + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 2); + + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 56); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestPrependOfVoid) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + auto pgmReturn = pb.NewEmptyListOfVoid(); + pgmReturn = pb.Prepend(pb.NewVoid(), pgmReturn); + pgmReturn = pb.Prepend(pb.NewVoid(), pgmReturn); + + const auto graph = setup.BuildGraph(pgmReturn); + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 0); + + const auto iterator = graph->GetValue().GetListIterator(); + UNIT_ASSERT(!iterator.Skip()); + UNIT_ASSERT(!iterator.Skip()); + } + + Y_UNIT_TEST_LLVM(TestForkedPrepend) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto list = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id)); + const auto list1 = pb.Prepend(pb.NewDataLiteral<ui32>(34), list); + const auto list2 = pb.Prepend(pb.NewDataLiteral<ui32>(56), list1); + const auto list3 = pb.Prepend(pb.NewDataLiteral<ui32>(78), list1); + const auto pgmReturn = pb.NewTuple({list2, list3}); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator1 = graph->GetValue().GetElement(0).GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator1.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 56); + UNIT_ASSERT(iterator1.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34); + UNIT_ASSERT(!iterator1.Next(item)); + UNIT_ASSERT(!iterator1.Next(item)); + + const auto iterator2 = graph->GetValue().GetElement(1).GetListIterator(); + UNIT_ASSERT(iterator2.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 78); + UNIT_ASSERT(iterator2.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34); + UNIT_ASSERT(!iterator2.Next(item)); + UNIT_ASSERT(!iterator2.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestAppendOfVoid) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + auto pgmReturn = pb.NewEmptyListOfVoid(); + pgmReturn = pb.Append(pgmReturn, pb.NewVoid()); + pgmReturn = pb.Append(pgmReturn, pb.NewVoid()); + + const auto graph = setup.BuildGraph(pgmReturn); + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 0); + + const auto iterator = graph->GetValue().GetListIterator(); + UNIT_ASSERT(!iterator.Skip()); + UNIT_ASSERT(!iterator.Skip()); + } + + Y_UNIT_TEST_LLVM(TestExtend) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto emptyList = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id)); + const auto list1 = pb.Append(emptyList, pb.NewDataLiteral<ui32>(34)); + const auto list2 = pb.Append(emptyList, pb.NewDataLiteral<ui32>(56)); + const auto pgmReturn = pb.Extend(list1, list2); + + const auto graph = setup.BuildGraph(pgmReturn); + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 2); + + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 56); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestExtend3) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto emptyList = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id)); + const auto list1 = pb.Append(emptyList, pb.NewDataLiteral<ui32>(34)); + const auto list2 = pb.Append(emptyList, pb.NewDataLiteral<ui32>(56)); + const auto list3 = pb.Append(emptyList, pb.NewDataLiteral<ui32>(7)); + const auto list4 = pb.Append(list3, pb.NewDataLiteral<ui32>(12)); + const auto pgmReturn = pb.Extend({ list1, emptyList, list2, list4 }); + + const auto graph = setup.BuildGraph(pgmReturn); + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 4); + + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 56); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 7); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 12); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestExtendOverFlows) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral(0.0); + const auto data1 = pb.NewDataLiteral(1.1); + const auto data2 = pb.NewDataLiteral(-3.14); + const auto data3 = pb.NewDataLiteral(121324.323); + const auto data4 = pb.NewDataLiteral(-7898.8); + const auto type = pb.NewDataType(NUdf::TDataType<double>::Id); + const auto list1 = pb.NewList(type, {data0, data1, data2}); + const auto list2 = pb.NewList(type, {data3, data4}); + + const auto pgmReturn = pb.FromFlow(pb.Extend({pb.ToFlow(list2), pb.ToFlow(list1), pb.ToFlow(list2)})); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 121324.323); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -7898.8); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.0); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 1.1); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -3.14); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 121324.323); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -7898.8); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestStreamForwardList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto data1 = pb.NewDataLiteral<ui32>(34); + const auto data2 = pb.NewDataLiteral<ui32>(56); + const auto data3 = pb.NewDataLiteral<ui32>(7); + const auto type = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto list = pb.NewList(type, {data1, data2, data3}); + + const auto pgmReturn = pb.ForwardList(pb.Iterator(list, {})); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 56); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 7); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFlowForwardList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto data1 = pb.NewDataLiteral<ui32>(34); + const auto data2 = pb.NewDataLiteral<ui32>(56); + const auto data3 = pb.NewDataLiteral<ui32>(7); + const auto type = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto list = pb.NewList(type, {data1, data2, data3}); + + const auto pgmReturn = pb.ForwardList(pb.ToFlow(list)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 56); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 7); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFlowFromOptional) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral(0.0); + const auto data1 = pb.NewDataLiteral(1.1); + const auto data2 = pb.NewDataLiteral(-3.14); + const auto type = pb.NewDataType(NUdf::TDataType<double>::Id); + + const auto opt0 = pb.NewOptional(data0); + const auto opt1 = pb.NewOptional(data1); + const auto opt2 = pb.NewOptional(data2); + const auto opt = pb.NewEmptyOptional(pb.NewOptionalType(type)); + + + const auto pgmReturn = pb.FromFlow(pb.Extend({pb.ToFlow(opt0), pb.ToFlow(opt), pb.ToFlow(opt1), pb.ToFlow(opt), pb.ToFlow(opt2)})); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.0); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 1.1); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -3.14); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestCollectOverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral(0.0); + const auto data1 = pb.NewDataLiteral(1.1); + const auto data2 = pb.NewDataLiteral(-3.14); + const auto data3 = pb.NewDataLiteral(121324.323); + const auto data4 = pb.NewDataLiteral(-7898.8); + const auto type = pb.NewDataType(NUdf::TDataType<double>::Id); + const auto data = pb.NewList(type, {data0, data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.ToFlow(data)); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto list = graph->GetValue(); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(0).template Get<double>(), 0.0); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(3).template Get<double>(), 121324.323); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(1).template Get<double>(), 1.1); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(2).template Get<double>(), -3.14); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(4).template Get<double>(), -7898.8); + } + + Y_UNIT_TEST_LLVM(TestAdd) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto pgmReturn = pb.Add(data1, data2); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto res = graph->GetValue().template Get<ui32>(); + UNIT_ASSERT_VALUES_EQUAL(res, 3); + } + + Y_UNIT_TEST_LLVM(TestSub) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(7); + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto pgmReturn = pb.Sub(data1, data2); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto res = graph->GetValue().template Get<ui32>(); + UNIT_ASSERT_VALUES_EQUAL(res, 5); + } + + Y_UNIT_TEST_LLVM(TestMul) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(3); + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto pgmReturn = pb.Mul(data1, data2); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto res = graph->GetValue().template Get<ui32>(); + UNIT_ASSERT_VALUES_EQUAL(res, 6); + } + + Y_UNIT_TEST_LLVM(TestDiv) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(17); + const auto data2 = pb.NewDataLiteral<ui32>(3); + const auto pgmReturn = pb.Div(data1, data2); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto value = graph->GetValue(); + UNIT_ASSERT(value); + UNIT_ASSERT_VALUES_EQUAL(value.template Get<ui32>(), 5); + } + + Y_UNIT_TEST_LLVM(TestDivZero) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(17); + const auto data2 = pb.NewDataLiteral<ui32>(0); + const auto pgmReturn = pb.Div(data1, data2); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto value = graph->GetValue(); + UNIT_ASSERT(!value); + } + + Y_UNIT_TEST_LLVM(TestIDivOverflow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<i32>(Min<i32>()); + const auto data2 = pb.NewDataLiteral<i32>(-1); + const auto pgmReturn = pb.Div(data1, data2); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto value = graph->GetValue(); + UNIT_ASSERT(!value); + } + + Y_UNIT_TEST_LLVM(TestMod) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(17); + const auto data2 = pb.NewDataLiteral<ui32>(3); + const auto pgmReturn = pb.Mod(data1, data2); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto value = graph->GetValue(); + UNIT_ASSERT(value); + UNIT_ASSERT_VALUES_EQUAL(value.template Get<ui32>(), 2); + } + + Y_UNIT_TEST_LLVM(TestModZero) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(17); + const auto data2 = pb.NewDataLiteral<ui32>(0); + const auto pgmReturn = pb.Mod(data1, data2); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto value = graph->GetValue(); + UNIT_ASSERT(!value); + } + + Y_UNIT_TEST_LLVM(TestIModOverflow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<i32>(Min<i32>()); + const auto data2 = pb.NewDataLiteral<i32>(-1); + const auto pgmReturn = pb.Mod(data1, data2); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto value = graph->GetValue(); + UNIT_ASSERT(!value); + } + + Y_UNIT_TEST_LLVM(TestStruct) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto data3 = pb.NewDataLiteral<ui32>(3); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + auto structObj = pb.NewEmptyStruct(); + structObj = pb.AddMember(structObj, "x", data2); + structObj = pb.AddMember(structObj, "z", data1); + structObj = pb.AddMember(structObj, "y", data3); + const auto pgmReturn = pb.NewList(dataType, { + pb.Member(structObj, "x"), + pb.Member(structObj, "y"), + pb.Member(structObj, "z") + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestMapOverList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto data1 = pb.NewOptional(pb.NewDataLiteral<i32>(1)); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<i32>(2)); + const auto data3 = pb.NewOptional(pb.NewDataLiteral<i32>(3)); + const auto data4 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i32>::Id); + const auto list = pb.NewList(dataType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Sub(pb.Mul(item, data2), data1); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 5); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestMapOverStream) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i16>::Id)); + const auto data0 = pb.NewOptional(pb.NewDataLiteral<i16>(0)); + const auto data1 = pb.NewOptional(pb.NewDataLiteral<i16>(-1)); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<i16>(3)); + const auto data3 = pb.NewOptional(pb.NewDataLiteral<i16>(7)); + const auto data4 = pb.NewOptional(pb.NewDataLiteral<i16>(11)); + const auto data5 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i16>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5}); + + const auto pgmReturn = pb.Map(pb.Iterator(list, {}), + [&](TRuntimeNode item) { + return pb.Div(data3, item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), -7); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), 2); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), 1); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), 0); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestMapOverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("PREFIX:"); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very large string"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>(""); + const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("small"); + const auto type = pb.NewDataType(NUdf::EDataSlot::String); + const auto list = pb.NewList(type, {data1, data2, data3}); + + const auto pgmReturn = pb.FromFlow(pb.Map(pb.ToFlow(list), + [&](TRuntimeNode item) { + return pb.Concat(data0, item); + })); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNBOXED_VALUE_STR_EQUAL(item, "PREFIX:very large string"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNBOXED_VALUE_STR_EQUAL(item, "PREFIX:"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNBOXED_VALUE_STR_EQUAL(item, "PREFIX:small"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + +#ifndef _win_ + Y_UNIT_TEST_LLVM(TestMapUnwrapThrow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i8>::Id)); + const auto data1 = pb.NewOptional(pb.NewDataLiteral<i8>(1)); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<i8>(7)); + const auto data0 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i8>::Id); + const auto list = pb.LazyList(pb.NewList(dataType, {data1, data0})); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Sub(data2, pb.Unwrap(item, pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), 6); + UNIT_ASSERT_EXCEPTION(iterator.Next(item), yexception); + } + + Y_UNIT_TEST_LLVM(TestMapUnwrapThrowMessage) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i8>::Id)); + const auto data1 = pb.NewOptional(pb.NewDataLiteral<i8>(1)); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<i8>(7)); + const auto data0 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i8>::Id); + const auto list = pb.LazyList(pb.NewList(dataType, {data1, data0})); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Sub(data2, pb.Unwrap(item, pb.Concat( + pb.NewDataLiteral<NUdf::EDataSlot::String>("a"), + pb.NewDataLiteral<NUdf::EDataSlot::String>("b")), + "", 0, 0)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), 6); + UNIT_ASSERT_EXCEPTION(iterator.Next(item), yexception); + } + + Y_UNIT_TEST_LLVM(TestMapEnsureThrowMessage) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<i8>::Id); + const auto data1 = pb.NewDataLiteral<i8>(1); + const auto data2 = pb.NewDataLiteral<i8>(7); + const auto list = pb.LazyList(pb.NewList(dataType, {data1, data2})); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Sub(data2, + pb.Ensure(item, pb.Less(item, data2), pb.NewDataLiteral<NUdf::EDataSlot::String>("BAD"),"", 0, 0)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), 6); + UNIT_ASSERT_EXCEPTION(iterator.Next(item), yexception); + } +#endif + Y_UNIT_TEST_LLVM(TestMapWithCoalesce) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i16>::Id)); + const auto data0 = pb.NewDataLiteral<i16>(0); + const auto data1 = pb.NewOptional(pb.NewDataLiteral<i16>(1)); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<i16>(2)); + const auto data3 = pb.NewOptional(pb.NewDataLiteral<i16>(3)); + const auto data = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i16>::Id); + const auto list = pb.NewList(dataType, {data1, data2, data3, data}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Coalesce(item, data0); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), 0); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestSizeOfOptional) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i16>::Id)); + const auto data0 = pb.NewOptional(pb.NewDataLiteral<i16>(0)); + const auto data1 = pb.NewOptional(pb.NewDataLiteral<i16>(1)); + const auto data = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i16>::Id); + const auto list = pb.NewList(dataType, {data1, data0, data}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Size(item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestSizeOfOptionalString) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<char*>::Id)); + const auto data0 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("0123456789")); + const auto data1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("XYZ")); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("")); + const auto data3 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("qwertyuioplkjhgfdsazxcvbnm")); + const auto data = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Size(pb.Concat(item, data0)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 20U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 13U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 10U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 36U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestMapWithIfExists) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<ui16>::Id)); + const auto data0 = pb.NewDataLiteral<ui16>(666); + const auto data1 = pb.NewOptional(pb.NewDataLiteral<ui16>(1)); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<ui16>(2)); + const auto data3 = pb.NewOptional(pb.NewDataLiteral<ui16>(3)); + const auto data = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui16>::Id); + const auto list = pb.NewList(dataType, {data1, data2, data3, data}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.If(pb.Exists(item), pb.Increment(pb.Unwrap(item, pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0)), data0); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 666); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestMapOverListLazy) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1U); + const auto data2 = pb.NewDataLiteral<ui32>(2U); + const auto data3 = pb.NewDataLiteral<ui32>(3U); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto list = pb.NewList(dataType, {data1, data2, data3}); + + const auto pgmReturn = pb.Map(pb.LazyList(list), + [&](TRuntimeNode item) { + return pb.Add(pb.Mul(item, data2), data1); + } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 5); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 7); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestMapOverOptional) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto list = pb.NewOptional(data1); + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Add(pb.Mul(item, data2), data1); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto value = graph->GetValue(); + UNIT_ASSERT(value); + UNIT_ASSERT_VALUES_EQUAL(value.template Get<ui32>(), 3); + } + + Y_UNIT_TEST_LLVM(TestFloatMinMax) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<float>(-NAN); + const auto data2 = pb.NewDataLiteral<float>(HUGE_VALF); + const auto data3 = pb.NewDataLiteral<float>(3.14f); + const auto data4 = pb.NewDataLiteral<float>(-2.13f); + const auto data5 = pb.NewDataLiteral<float>(-HUGE_VALF); + const auto dataType = pb.NewDataType(NUdf::TDataType<float>::Id); + const auto list = pb.NewList(dataType, {data1, data2, data3, data4, data5}); + const auto pgmReturn = pb.FlatMap(list, + [&](TRuntimeNode left) { + return pb.Map(list, + [&](TRuntimeNode right) { + return pb.NewTuple({pb.Min(left, right), pb.Max(left, right)}); + }); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(std::isnan(item.GetElement(0).template Get<float>())); + UNIT_ASSERT(std::isnan(item.GetElement(1).template Get<float>())); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), HUGE_VALF); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), 3.14f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), 3.14f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -2.13f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), -2.13f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), -HUGE_VALF); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), HUGE_VALF); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), HUGE_VALF); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), 3.14f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), HUGE_VALF); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -2.13f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), HUGE_VALF); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), HUGE_VALF); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), 3.14f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), 3.14f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), 3.14f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), HUGE_VALF); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), 3.14f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), 3.14f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -2.13f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), 3.14f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), 3.14f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -2.13f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), -2.13f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -2.13f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), HUGE_VALF); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -2.13f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), 3.14f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -2.13f); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), -2.13f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), -2.13f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), -HUGE_VALF); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), HUGE_VALF); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), 3.14f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), -2.13f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -HUGE_VALF); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), -HUGE_VALF); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFloatMod) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<float>(-1.75f); + const auto data2 = pb.NewDataLiteral<float>(3.14f); + const auto data3 = pb.NewDataLiteral<float>(-6.28f); + const auto data4 = pb.NewDataLiteral<float>(7.28f); + const auto dataType = pb.NewDataType(NUdf::TDataType<float>::Id); + const auto list = pb.NewList(dataType, {data1, data2, data3, data4}); + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Mod(item, data2); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -1.75f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 0.0f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 0.0f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 1.0f); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestDoubleMod) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<double>(-1.75); + const auto data2 = pb.NewDataLiteral<double>(3.14); + const auto data3 = pb.NewDataLiteral<double>(-6.28); + const auto data4 = pb.NewDataLiteral<double>(7.28); + const auto dataType = pb.NewDataType(NUdf::TDataType<double>::Id); + const auto list = pb.NewList(dataType, {data1, data2, data3, data4}); + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Mod(item, data2); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -1.75); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 1.0); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestDiscardOverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000"); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200"); + const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300"); + const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3}); + + const auto pgmReturn = pb.FromFlow(pb.Discard(pb.ToFlow(list))); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestHead) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto itemType = pb.NewDataType(NUdf::TDataType<float>::Id); + const auto listType = pb.NewListType(itemType); + + const auto data0 = pb.NewEmptyList(itemType); + const auto data1 = pb.NewList(itemType, {pb.NewDataLiteral<float>(-1.5f), pb.NewDataLiteral<float>(0.f), pb.NewDataLiteral<float>(3.14f)}); + const auto data2 = pb.NewList(itemType, {pb.NewDataLiteral<float>(3.14f), pb.NewDataLiteral<float>(-1.5f), pb.NewDataLiteral<float>(0.f)}); + const auto data3 = pb.LazyList(pb.NewList(itemType, {pb.NewDataLiteral<float>(1.1f), pb.NewDataLiteral<float>(-2.2f)})); + + const auto list = pb.NewList(listType, {data0, data1, data2, data3}); + + const auto pgmReturn = pb.Map(list, [&](TRuntimeNode item) { return pb.Head(item); }); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -1.5f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 3.14f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 1.1f); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestLast) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto itemType = pb.NewDataType(NUdf::TDataType<float>::Id); + const auto listType = pb.NewListType(itemType); + + const auto data0 = pb.NewEmptyList(itemType); + const auto data1 = pb.NewList(itemType, {pb.NewDataLiteral<float>(-1.5f), pb.NewDataLiteral<float>(0.f), pb.NewDataLiteral<float>(3.14f)}); + const auto data2 = pb.NewList(itemType, {pb.NewDataLiteral<float>(3.14f), pb.NewDataLiteral<float>(-1.5f), pb.NewDataLiteral<float>(0.f)}); + const auto data3 = pb.LazyList(pb.NewList(itemType, {pb.NewDataLiteral<float>(1.1f), pb.NewDataLiteral<float>(-2.2f)})); + + const auto list = pb.NewList(listType, {data0, data1, data2, data3}); + + const auto pgmReturn = pb.Map(list, [&](TRuntimeNode item) { return pb.Last(item); }); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 3.14f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 0.f); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -2.2f); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestCoalesce) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto data3 = pb.NewDataLiteral<ui32>(3); + auto pgmReturn = pb.NewEmptyStruct(); + pgmReturn = pb.AddMember(pgmReturn, "A", pb.Coalesce( + pb.NewOptional(data1), data2)); + pgmReturn = pb.AddMember(pgmReturn, "B", pb.Coalesce( + pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id), data2)); + pgmReturn = pb.AddMember(pgmReturn, "C", pb.Coalesce( + pb.NewOptional(data2), pb.NewOptional(data3))); + pgmReturn = pb.AddMember(pgmReturn, "D", pb.Coalesce( + pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id), pb.NewOptional(data3))); + pgmReturn = pb.AddMember(pgmReturn, "E", pb.Coalesce( + pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id), pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id))); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto value = graph->GetValue(); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(0).template Get<ui32>(), 1); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(1).template Get<ui32>(), 2); + UNIT_ASSERT(value.GetElement(2)); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(2).template Get<ui32>(), 2); + UNIT_ASSERT(value.GetElement(3)); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(3).template Get<ui32>(), 3); + UNIT_ASSERT(!value.GetElement(4)); + } + + Y_UNIT_TEST_LLVM(TestExists) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui64>(1); + const auto optionalType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<ui64>::Id)); + const auto list = pb.NewList(optionalType, {pb.NewOptional(data1), pb.NewEmptyOptional(optionalType)}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Exists(item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestIf) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto truth = pb.NewDataLiteral(true); + const auto falsehood = pb.NewDataLiteral(false); + const auto list = pb.NewList(pb.NewDataType(NUdf::TDataType<bool>::Id), {truth, falsehood}); + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.If(item, data1, data2); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestIfPresent) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto filled = pb.NewOptional(data1); + const auto empty = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); + const auto list = pb.NewList(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<ui32>::Id)), {filled, empty}); + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.IfPresent({item}, + [&](TRuntimeNode::TList unpacked){ + return unpacked.front(); + }, data2); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestIfPresentTwo) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewOptional(pb.NewDataLiteral<i8>(+1)); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<i8>(-1)); + const auto empty1 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i8>::Id); + + const auto data3 = pb.NewOptional(pb.NewDataLiteral<bool>(true)); + const auto data4= pb.NewOptional(pb.NewDataLiteral<bool>(false)); + const auto empty2 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id); + + const auto list1 = pb.NewList(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i8>::Id)), {data1, empty1, data2}); + const auto list2 = pb.NewList(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<bool>::Id)), {data3, empty2, data4}); + + const auto pgmReturn = pb.FlatMap(list1, + [&](TRuntimeNode item1) { + return pb.FlatMap(list2, + [&](TRuntimeNode item2) { + return pb.IfPresent({item1, item2}, + [&](TRuntimeNode::TList unpacked) { + return pb.NewOptional(pb.If(unpacked.back(), pb.Minus(unpacked.front()), unpacked.front())); + }, empty1); + }); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), -1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), +1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), +1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), -1); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestIfPresentThree) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewOptional(pb.NewDataLiteral<i8>(+1)); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<i8>(-1)); + const auto empty1 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i8>::Id); + + const auto data3 = pb.NewOptional(pb.NewDataLiteral<bool>(true)); + const auto data4= pb.NewOptional(pb.NewDataLiteral<bool>(false)); + const auto empty2 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id); + + const auto data5 = pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i8>(5))); + const auto data6 = pb.NewOptional(pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i8>::Id)); + const auto type2 = pb.NewOptionalType(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i8>::Id))); + const auto empty3 = pb.NewEmptyOptional(type2); + + const auto list1 = pb.NewList(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i8>::Id)), {data1, empty1, data2}); + const auto list2 = pb.NewList(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<bool>::Id)), {data3, empty2, data4}); + const auto list3 = pb.NewList(type2, {data5, empty3, data6}); + + const auto pgmReturn = pb.FlatMap(list1, + [&](TRuntimeNode item1) { + return pb.FlatMap(list2, + [&](TRuntimeNode item2) { + return pb.FlatMap(list3, + [&](TRuntimeNode item3) { + return pb.IfPresent({item1, item2, item3}, + [&](TRuntimeNode::TList unpacked) { + return pb.NewOptional(pb.If(unpacked[1], + pb.Add(unpacked.front(), pb.Coalesce(unpacked.back(), pb.NewDataLiteral<i8>(3))), + pb.Sub(unpacked.front(), pb.Coalesce(unpacked.back(), pb.NewDataLiteral<i8>(7))) + )); + }, empty1); + }); + }); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), +6); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), +4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), -4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), -6); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), +4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), +2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), -6); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), -8); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestIfPresentSame) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewOptional(pb.NewDataLiteral<i8>(+1)); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<i8>(-1)); + const auto data0 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i8>::Id); + + const auto list = pb.NewList(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i8>::Id)), {data1, data0, data2}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + const auto minus = pb.Minus(item); + return pb.NewTuple({ + pb.IfPresent({minus}, + [&](TRuntimeNode::TList unpacked) { + return pb.Abs(unpacked.front()); + }, data0), + pb.IfPresent({minus}, + [&](TRuntimeNode::TList unpacked) { + return pb.Minus(unpacked.front()); + }, data0), + }); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i8>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i8>(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i8>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i8>(), -1); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestIncDec) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(7); + auto pgmReturn = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id)); + pgmReturn = pb.Append(pgmReturn, pb.Increment(data1)); + pgmReturn = pb.Append(pgmReturn, pb.Decrement(data1)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 8); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 6); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestLogical) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto truth = pb.NewDataLiteral(true); + const auto falsehood = pb.NewDataLiteral(false); + auto pgmReturn = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<bool>::Id)); + + pgmReturn = pb.Append(pgmReturn, pb.And({truth, truth})); + pgmReturn = pb.Append(pgmReturn, pb.And({truth, falsehood})); + pgmReturn = pb.Append(pgmReturn, pb.And({falsehood, truth})); + pgmReturn = pb.Append(pgmReturn, pb.And({falsehood, falsehood})); + + pgmReturn = pb.Append(pgmReturn, pb.Or({truth, truth})); + pgmReturn = pb.Append(pgmReturn, pb.Or({falsehood, truth})); + pgmReturn = pb.Append(pgmReturn, pb.Or({truth, falsehood})); + pgmReturn = pb.Append(pgmReturn, pb.Or({falsehood, falsehood})); + + pgmReturn = pb.Append(pgmReturn, pb.Xor({truth, truth})); + pgmReturn = pb.Append(pgmReturn, pb.Xor({falsehood, truth})); + pgmReturn = pb.Append(pgmReturn, pb.Xor({truth, falsehood})); + pgmReturn = pb.Append(pgmReturn, pb.Xor({falsehood, falsehood})); + + pgmReturn = pb.Append(pgmReturn, pb.Not(truth)); + pgmReturn = pb.Append(pgmReturn, pb.Not(falsehood)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue item; + + // and + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + + // or + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + + // xor + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + + // not + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestZip) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), { + pb.NewDataLiteral<ui32>(34), + pb.NewDataLiteral<ui32>(56) + }); + const auto list2 = pb.NewList(pb.NewDataType(NUdf::TDataType<char*>::Id), { + pb.NewDataLiteral<NUdf::EDataSlot::String>("Q"), + pb.NewDataLiteral<NUdf::EDataSlot::String>("E"), + pb.NewDataLiteral<NUdf::EDataSlot::String>("W") + }); + + const auto pgmReturn = pb.Zip({list1, list2}); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto list = graph->GetValue(); + + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 2U); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(0).GetElement(0).template Get<ui32>(), 34); + UNBOXED_VALUE_STR_EQUAL(list.GetElement(0).GetElement(1), "Q"); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(1).GetElement(0).template Get<ui32>(), 56); + UNBOXED_VALUE_STR_EQUAL(list.GetElement(1).GetElement(1), "E"); + } + + Y_UNIT_TEST_LLVM(TestZipLazy) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), { + pb.NewDataLiteral<ui32>(34), + pb.NewDataLiteral<ui32>(56) + }); + const auto list2 = pb.NewList(pb.NewDataType(NUdf::TDataType<char*>::Id), { + pb.NewDataLiteral<NUdf::EDataSlot::String>("Q"), + pb.NewDataLiteral<NUdf::EDataSlot::String>("E"), + pb.NewDataLiteral<NUdf::EDataSlot::String>("W") + }); + + const auto pgmReturn = pb.Zip({pb.LazyList(list1), list2}); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 34); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "Q"); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 56); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "E"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestZipAll) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), { + pb.NewDataLiteral<ui32>(34), + pb.NewDataLiteral<ui32>(56) + }); + const auto list2 = pb.NewList(pb.NewDataType(NUdf::TDataType<char*>::Id), { + pb.NewDataLiteral<NUdf::EDataSlot::String>("Q"), + pb.NewDataLiteral<NUdf::EDataSlot::String>("E"), + pb.NewDataLiteral<NUdf::EDataSlot::String>("W") + }); + + const auto pgmReturn = pb.ZipAll({list1, list2}); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto list = graph->GetValue(); + + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 3U); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(0).GetElement(0).template Get<ui32>(), 34); + UNBOXED_VALUE_STR_EQUAL(list.GetElement(0).GetElement(1), "Q"); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(1).GetElement(0).template Get<ui32>(), 56); + UNBOXED_VALUE_STR_EQUAL(list.GetElement(1).GetElement(1), "E"); + UNIT_ASSERT(!list.GetElement(2).GetElement(0)); + UNBOXED_VALUE_STR_EQUAL(list.GetElement(2).GetElement(1), "W"); + } + + Y_UNIT_TEST_LLVM(TestZipAllLazy) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), { + pb.NewDataLiteral<ui32>(34), + pb.NewDataLiteral<ui32>(56) + }); + const auto list2 = pb.NewList(pb.NewDataType(NUdf::TDataType<char*>::Id), { + pb.NewDataLiteral<NUdf::EDataSlot::String>("Q"), + pb.NewDataLiteral<NUdf::EDataSlot::String>("E"), + pb.NewDataLiteral<NUdf::EDataSlot::String>("W") + }); + + const auto pgmReturn = pb.ZipAll({list1, pb.LazyList(list2)}); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 34); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "Q"); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 56); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "E"); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "W"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestReduce) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1)); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2)); + const auto data3 = pb.NewOptional(pb.NewDataLiteral<ui32>(3)); + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<ui32>::Id)); + const auto list = pb.NewList(dataType, {data1, data2, data3}); + const auto empty = pb.AddMember(pb.AddMember( + pb.NewEmptyStruct(), "Min", pb.NewEmptyOptional(dataType)), + "Max", pb.NewEmptyOptional(dataType)); + const auto pgmReturn = pb.Reduce(list, empty, + [&](TRuntimeNode item, TRuntimeNode state1) { + return pb.AddMember( + pb.AddMember(pb.NewEmptyStruct(), "Min", pb.AggrMin(pb.Member(state1, "Min"), item)), + "Max", pb.AggrMax(pb.Member(state1, "Max"), item) + ); + }, + [&](TRuntimeNode state) { + return state; + }, empty, + [&](TRuntimeNode state1, TRuntimeNode state2) { + return pb.AddMember( + pb.AddMember(pb.NewEmptyStruct(), "Min", pb.AggrMin(pb.Member(state1, "Min"), pb.Member(state2, "Min"))) + , "Max", pb.AggrMax(pb.Member(state1, "Max"), pb.Member(state2, "Max")) + ); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto value = graph->GetValue(); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(1).template Get<ui32>(), 1); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(0).template Get<ui32>(), 3); + } + + Y_UNIT_TEST_LLVM(TestReduceOverStream) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1)); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2)); + const auto data3 = pb.NewOptional(pb.NewDataLiteral<ui32>(3)); + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<ui32>::Id)); + const auto list = pb.NewList(dataType, {data1, data2, data3}); + const auto empty = pb.AddMember(pb.AddMember( + pb.NewEmptyStruct(), "Min", pb.NewEmptyOptional(dataType)), + "Max", pb.NewEmptyOptional(dataType)); + const auto pgmReturn = pb.Reduce(pb.Iterator(list, {}), empty, + [&](TRuntimeNode item, TRuntimeNode state1) { + return pb.AddMember( + pb.AddMember(pb.NewEmptyStruct(), "Min", pb.AggrMin(pb.Member(state1, "Min"), item)), + "Max", pb.AggrMax(pb.Member(state1, "Max"), item) + ); + }, + [&](TRuntimeNode state) { + return state; + }, empty, + [&](TRuntimeNode state1, TRuntimeNode state2) { + return pb.AddMember( + pb.AddMember(pb.NewEmptyStruct(), "Min", pb.AggrMin(pb.Member(state1, "Min"), pb.Member(state2, "Min"))) + , "Max", pb.AggrMax(pb.Member(state1, "Max"), pb.Member(state2, "Max")) + ); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto value = graph->GetValue(); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(1).template Get<ui32>(), 1); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(0).template Get<ui32>(), 3); + } + + Y_UNIT_TEST_LLVM(TestListLength) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), + {pb.NewDataLiteral<ui32>(34), pb.NewDataLiteral<ui32>(56)}); + const auto pgmReturn = pb.Length(list); + + const auto graph = setup.BuildGraph(pgmReturn); + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().template Get<ui64>(), 2); + } + + Y_UNIT_TEST_LLVM(TestReverse) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), + {pb.NewDataLiteral<ui32>(34), pb.NewDataLiteral<ui32>(56)}); + const auto pgmReturn = pb.Reverse(list); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 56); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestSkipForAppend) { + const ui32 n = 100; + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + for (ui32 i = 0; i < n; ++i) { + auto list = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id)); + for (ui32 j = 0; j < n; ++j) { + list = pb.Append(list, pb.NewDataLiteral(j)); + } + + const auto skippedList = pb.Skip(list, pb.NewDataLiteral<ui64>(i)); + auto changedList = skippedList; + for (ui32 j = 0; j < n; ++j) { + changedList = pb.Prepend(pb.NewDataLiteral(n + n - 1 - j), changedList); + } + + auto pgmReturn = pb.NewEmptyList(list.GetStaticType()); + pgmReturn = pb.Append(pgmReturn, list); + pgmReturn = pb.Append(pgmReturn, skippedList); + pgmReturn = pb.Append(pgmReturn, changedList); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iteratorLists = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue test, item; + + UNIT_ASSERT(iteratorLists.Next(test)); + auto iterator = test.GetListIterator(); + for (ui32 j = 0; j < n; ++j) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j); + } + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + + UNIT_ASSERT(iteratorLists.Next(test)); + iterator = test.GetListIterator(); + for (ui32 j = i; j < n; ++j) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j); + } + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + + UNIT_ASSERT(iteratorLists.Next(test)); + iterator = test.GetListIterator(); + for (ui32 j = 0; j < n; ++j) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j + n); + } + + for (ui32 j = i; j < n; ++j) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j); + } + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iteratorLists.Next(test)); + UNIT_ASSERT(!iteratorLists.Next(test)); + } + } + + Y_UNIT_TEST_LLVM(TestSkipForPrepend) { + const ui32 n = 100; + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + for (ui32 i = 0; i < n; ++i) { + auto list = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id)); + for (ui32 j = 0; j < n; ++j) { + list = pb.Prepend(pb.NewDataLiteral(n - 1 - j), list); + } + + const auto skippedList = pb.Skip(list, pb.NewDataLiteral<ui64>(i)); + auto changedList = skippedList; + for (ui32 j = 0; j < n; ++j) { + changedList = pb.Prepend(pb.NewDataLiteral(n + n - 1 - j), changedList); + } + + const auto pgmReturn = pb.NewList(list.GetStaticType(), {list, skippedList, changedList}); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iteratorLists = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue test, item; + + UNIT_ASSERT(iteratorLists.Next(test)); + auto iterator = test.GetListIterator(); + for (ui32 j = 0; j < n; ++j) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j); + } + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + + UNIT_ASSERT(iteratorLists.Next(test)); + iterator = test.GetListIterator(); + for (ui32 j = i; j < n; ++j) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j); + } + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + + UNIT_ASSERT(iteratorLists.Next(test)); + iterator = test.GetListIterator(); + for (ui32 j = 0; j < n; ++j) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j + n); + } + + for (ui32 j = i; j < n; ++j) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j); + } + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iteratorLists.Next(test)); + UNIT_ASSERT(!iteratorLists.Next(test)); + } + } + + Y_UNIT_TEST_LLVM(TestTakeForAppend) { + const ui32 n = 100; + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + for (ui32 i = 0; i < n; ++i) { + auto list = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id)); + for (ui32 j = 0; j < n; ++j) { + list = pb.Append(list, pb.NewDataLiteral(j)); + } + + const auto skippedList = pb.Take(list, pb.NewDataLiteral<ui64>(i)); + auto changedList = skippedList; + for (ui32 j = 0; j < n; ++j) { + changedList = pb.Append(changedList, pb.NewDataLiteral(n + j)); + } + + const auto pgmReturn = pb.NewList(list.GetStaticType(), {list, skippedList, changedList}); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iteratorLists = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue test, item; + + UNIT_ASSERT(iteratorLists.Next(test)); + auto iterator = test.GetListIterator(); + for (ui32 j = 0; j < n; ++j) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j); + } + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + + UNIT_ASSERT(iteratorLists.Next(test)); + iterator = test.GetListIterator(); + for (ui32 j = 0; j < i; ++j) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j); + } + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + + UNIT_ASSERT(iteratorLists.Next(test)); + iterator = test.GetListIterator(); + for (ui32 j = 0; j < i; ++j) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j); + } + + for (ui32 j = 0; j < n; ++j) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j + n); + } + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iteratorLists.Next(test)); + UNIT_ASSERT(!iteratorLists.Next(test)); + } + } + + Y_UNIT_TEST_LLVM(TestTakeForPrepend) { + const ui32 n = 100; + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + for (ui32 i = 0; i < n; ++i) { + auto list = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id)); + for (ui32 j = 0; j < n; ++j) { + list = pb.Prepend(pb.NewDataLiteral(n - 1- j), list); + } + + const auto skippedList = pb.Take(list, pb.NewDataLiteral<ui64>(i)); + auto changedList = skippedList; + for (ui32 j = 0; j < n; ++j) { + changedList = pb.Append(changedList, pb.NewDataLiteral(n + j)); + } + + const auto pgmReturn = pb.NewList(list.GetStaticType(), {list, skippedList, changedList}); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iteratorLists = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue test, item; + + UNIT_ASSERT(iteratorLists.Next(test)); + auto iterator = test.GetListIterator(); + for (ui32 j = 0; j < n; ++j) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j); + } + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + + UNIT_ASSERT(iteratorLists.Next(test)); + iterator = test.GetListIterator(); + for (ui32 j = 0; j < i; ++j) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j); + } + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + + UNIT_ASSERT(iteratorLists.Next(test)); + iterator = test.GetListIterator(); + for (ui32 j = 0; j < i; ++j) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j); + } + + for (ui32 j = 0; j < n; ++j) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j + n); + } + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iteratorLists.Next(test)); + UNIT_ASSERT(!iteratorLists.Next(test)); + } + } + + Y_UNIT_TEST_LLVM(TestReplicate) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto pgmReturn = pb.Replicate(pb.NewDataLiteral<ui32>(34), + pb.NewDataLiteral<ui64>(4), "", 0, 0); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + for (ui32 i = 0; i < 4; ++i) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34); + } + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestIntegralCasts) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(258); + const auto pgmReturn = pb.Convert(data1, pb.NewDataType(NUdf::TDataType<ui8>::Id)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto res = graph->GetValue().template Get<ui8>(); + UNIT_ASSERT_VALUES_EQUAL(res, 2); + } + + Y_UNIT_TEST_LLVM(TestCastFourIntoTrue) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(4); + const auto pgmReturn = pb.Convert(data1, pb.NewDataType(NUdf::TDataType<bool>::Id)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto res = graph->GetValue().template Get<bool>(); + UNIT_ASSERT(res); + } + + Y_UNIT_TEST_LLVM(TestSubstring) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("aabbccc"); + const auto start1 = pb.NewDataLiteral<ui32>(3); + const auto count1 = pb.NewDataLiteral<ui32>(2); + const auto start2 = pb.NewDataLiteral<ui32>(4); + const auto count2 = pb.NewDataLiteral<ui32>(10); + const auto start3 = pb.NewDataLiteral<ui32>(10); + const auto count3 = pb.NewDataLiteral<ui32>(1); + const auto start4 = pb.NewDataLiteral<ui32>(0); + const auto count4 = pb.NewDataLiteral<ui32>(3); + + const auto dataType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<ui32>::Id)}); + const auto list = pb.NewList(dataType, { + pb.NewTuple({start1, count1}), + pb.NewTuple({start2, count2}), + pb.NewTuple({start3, count3}), + pb.NewTuple({start4, count4}) + }); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Substring(data1, pb.Nth(item, 0), pb.Nth(item, 1)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "bc"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "ccc"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, ""); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "aab"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestSubstringOptionalArgs) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto null = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); + const auto data0 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<const char*>::Id); + const auto data1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("")); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("0123456789")); + const auto data3 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("abcdefghijklmnopqrstuvwxyz")); + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<const char*>::Id)); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.NewTuple({ + pb.Substring(item, null, null), + pb.Substring(item, pb.NewOptional(pb.NewDataLiteral<ui32>(7U)), null), + pb.Substring(item, null, pb.NewOptional(pb.NewDataLiteral<ui32>(6U))), + pb.Substring(item, pb.NewOptional(pb.NewDataLiteral<ui32>(3U)), pb.NewOptional(pb.NewDataLiteral<ui32>(17U))) + }); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0U)); + UNIT_ASSERT(!item.GetElement(1U)); + UNIT_ASSERT(!item.GetElement(2U)); + UNIT_ASSERT(!item.GetElement(3U)); + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), ""); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), ""); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), ""); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(3U), ""); + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "0123456789"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "789"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "012345"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(3U), "3456789"); + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "abcdefghijklmnopqrstuvwxyz"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "hijklmnopqrstuvwxyz"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "abcdef"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(3U), "defghijklmnopqrst"); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFindAndSubstring) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto type = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<const char*>::Id)); + const auto data0 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<const char*>::Id); + const auto data1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("")); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("_/>_/>aab*SEP*bccc")); + const auto data3 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("*SEP*_/>a>><<___*SEP*a_/>b54b*SEP*c_/>3434cc*SEP*")); + const auto data4 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("(none)")); + + const auto sep1 = pb.NewDataLiteral<NUdf::EDataSlot::String>(""); + const auto sep2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("_/>"); + const auto sep3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("*SEP*"); + + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4}); + const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<const char*>::Id), {sep1, sep2, sep3}); + const auto null = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id); + + const auto pgmReturn = pb.FlatMap(list, + [&](TRuntimeNode data) { + return pb.Map(list1, + [&](TRuntimeNode sep) { + const auto first = pb.Find(data, sep, null); + const auto last = pb.RFind(data, sep, null); + const auto len = pb.Size(sep); + return pb.NewTuple({ + pb.Substring(data, null, first), + pb.Substring(data, pb.Add(first, len), pb.If(pb.Less(first, last), pb.Sub(last, pb.Add(first, len)), pb.NewDataLiteral<ui32>(0U))), + pb.Substring(data, pb.Add(last, len), null) + }); + }); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0U)); + UNIT_ASSERT(!item.GetElement(1U)); + UNIT_ASSERT(!item.GetElement(2U)); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0U)); + UNIT_ASSERT(!item.GetElement(1U)); + UNIT_ASSERT(!item.GetElement(2U)); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0U)); + UNIT_ASSERT(!item.GetElement(1U)); + UNIT_ASSERT(!item.GetElement(2U)); + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), ""); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), ""); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), ""); + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), ""); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), ""); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), ""); + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), ""); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), ""); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), ""); + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), ""); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "_/>_/>aab*SEP*bccc"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), ""); + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), ""); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), ""); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "aab*SEP*bccc"); + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "_/>_/>aab"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), ""); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "bccc"); + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), ""); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "*SEP*_/>a>><<___*SEP*a_/>b54b*SEP*c_/>3434cc*SEP*"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), ""); + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "*SEP*"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "a>><<___*SEP*a_/>b54b*SEP*c"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "3434cc*SEP*"); + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), ""); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "_/>a>><<___*SEP*a_/>b54b*SEP*c_/>3434cc"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), ""); + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), ""); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "(none)"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), ""); + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "(none)"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "(none)"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "(none)"); + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "(none)"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "(none)"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "(none)"); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFindFromPos) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data = pb.NewDataLiteral<NUdf::EDataSlot::String>("_</|>0123456789</|></|>abcxyz</|>*"); + const auto sep = pb.NewDataLiteral<NUdf::EDataSlot::String>("</|>"); + + const auto list = pb.ListFromRange(pb.NewDataLiteral<ui32>(0U), pb.Size(data), pb.NewDataLiteral<ui32>(2U)); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.NewTuple({pb.Find(data, sep, item), pb.RFind(data, sep, item)}); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 1U); + UNIT_ASSERT(!item.GetElement(1U)); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 15U); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 1U); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 15U); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 1U); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 15U); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 1U); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 15U); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 1U); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 15U); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 1U); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 15U); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 1U); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 15U); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 1U); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 19U); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 15U); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 19U); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 15U); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 29U); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 19U); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 29U); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 19U); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 29U); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 19U); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 29U); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 19U); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 29U); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 19U); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0U)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 29U); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0U)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 29U); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestListFromRange) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<ui16>(0); + const auto data1 = pb.NewDataLiteral<ui16>(9); + const auto data2 = pb.NewDataLiteral<ui16>(1); + const auto data3 = pb.NewDataLiteral<ui16>(2); + const auto data4 = pb.NewDataLiteral<ui16>(3); + + const auto list = pb.NewList(pb.NewDataType(NUdf::TDataType<ui16>::Id), {data2, data3, data4}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + const auto l = pb.ListFromRange(data0, data1, item); + return pb.NewTuple({pb.HasItems(l), pb.Length(l), pb.Head(l), pb.Last(l)}); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0U).template Get<bool>()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui64>(), 9ULL); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2U).template Get<ui16>(), 0U); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(3U).template Get<ui16>(), 8U); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0U).template Get<bool>()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui64>(), 5ULL); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2U).template Get<ui16>(), 0U); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(3U).template Get<ui16>(), 8U); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0U).template Get<bool>()); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui64>(), 3ULL); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2U).template Get<ui16>(), 0U); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(3U).template Get<ui16>(), 6U); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestSize) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("aaa"); + const auto data2 = pb.NewDataLiteral<ui32>(3); + const auto data3 = pb.NewDataLiteral<ui64>(7878786987536ull); + const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("qqqqq"); + const auto pgmReturn = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), + {pb.Size(data1), pb.Size(data2), pb.Size(data3), pb.Size(data4)}); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 8); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 5); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestEnumerate) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), + {pb.NewDataLiteral<ui32>(34), pb.NewDataLiteral<ui32>(56)}); + const auto pgmReturn = pb.Enumerate(list1); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 34); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 56); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestEnumerateLazy) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), + {pb.NewDataLiteral<ui32>(34), pb.NewDataLiteral<ui32>(56)}); + const auto pgmReturn = pb.Enumerate(pb.LazyList(list1)); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 34); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 56); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestEnumerateLazyThenReverse) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), + {pb.NewDataLiteral<ui32>(34), pb.NewDataLiteral<ui32>(56)}); + const auto pgmReturn = pb.Reverse(pb.Enumerate(pb.LazyList(list1))); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 56); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 34); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestEnumerateLazyThenSkip) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), + {pb.NewDataLiteral<ui32>(34), pb.NewDataLiteral<ui32>(56)}); + const auto one = pb.NewDataLiteral<ui64>(1); + const auto list = pb.Enumerate(pb.LazyList(list1)); + const auto skip = pb.Skip(list, one); + const auto pgmReturn = pb.NewTuple({skip, pb.Length(list), pb.Length(skip)}); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue().GetElement(0).GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 56); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetElement(1).template Get<ui64>(), 2); + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetElement(2).template Get<ui64>(), 1); + } + + Y_UNIT_TEST_LLVM(TestEnumerateLazyThenTake) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), + {pb.NewDataLiteral<ui32>(34), pb.NewDataLiteral<ui32>(56)}); + const auto one = pb.NewDataLiteral<ui64>(1); + const auto list = pb.Enumerate(pb.LazyList(list1)); + const auto take = pb.Take(list, one); + const auto pgmReturn = pb.NewTuple({take, pb.Length(list), pb.Length(take)}); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue().GetElement(0).GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 34); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetElement(1).template Get<ui64>(), 2); + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetElement(2).template Get<ui64>(), 1); + } + + template<bool LLVM> + void TestSortImpl(bool asc) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(3); + + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("aaa"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>(""); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("qqq"); + + const auto keyType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto payloadType = pb.NewDataType(NUdf::TDataType<char*>::Id); + auto structType = pb.NewEmptyStructType(); + structType = pb.NewStructType(structType, "payload", payloadType); + structType = pb.NewStructType(structType, "key", keyType); + + std::vector<std::pair<std::string_view, TRuntimeNode>> map1 = { + { "key", key1 }, + { "payload", payload1 } + }; + + std::vector<std::pair<std::string_view, TRuntimeNode>> map2 = { + { "key", key2 }, + { "payload", payload2 } + }; + + std::vector<std::pair<std::string_view, TRuntimeNode>> map3 = { + { "key", key3 }, + { "payload", payload3 } + }; + + const auto list = pb.NewList(structType, { + pb.NewStruct(map2), + pb.NewStruct(map1), + pb.NewStruct(map3) + }); + + { + const auto pgmReturn = pb.Sort(list, pb.NewDataLiteral(asc), + [&](TRuntimeNode item) { + return pb.Member(item, "key"); + }); + + if (asc) { + // ascending sort + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 1); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "aaa"); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 2); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), ""); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 3); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "qqq"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } else { + // descending sort + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 3); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "qqq"); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 2); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), ""); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 1); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "aaa"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + } + } + + Y_UNIT_TEST_LLVM(TestSort) { + TestSortImpl<LLVM>(true); + TestSortImpl<LLVM>(false); + } + + using TTriple = std::tuple<ui32, ui32, ui32>; + + TRuntimeNode TupleOrder(TProgramBuilder& pb, bool asc1, bool asc2, bool asc3) { + TVector<TRuntimeNode> ascending(3); + ascending[0] = pb.NewDataLiteral(asc1); + ascending[1] = pb.NewDataLiteral(asc2); + ascending[2] = pb.NewDataLiteral(asc3); + + TVector<TType*> tupleTypes(3); + tupleTypes[0] = pb.NewDataType(NUdf::TDataType<bool>::Id); + tupleTypes[1] = pb.NewDataType(NUdf::TDataType<bool>::Id); + tupleTypes[2] = pb.NewDataType(NUdf::TDataType<bool>::Id); + + return pb.NewTuple(pb.NewTupleType(tupleTypes), ascending); + } + + template<bool LLVM> + TVector<TTriple> SortTuples(TSetup<LLVM>& setup, TRuntimeNode list, TRuntimeNode order) + { + auto& pb = *setup.PgmBuilder; + const auto pgmReturn = pb.Sort(list, order, [](TRuntimeNode item) { return item; }); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + + TVector<TTriple> result; + for (NUdf::TUnboxedValue value; iterator.Next(value);) { + ui32 first = value.GetElement(0).template Get<ui32>(); + ui32 second = value.GetElement(1).template Get<ui32>(); + ui32 third = value.GetElement(2).template Get<ui32>(); + result.push_back(TTriple{ first, second, third }); + } + UNIT_ASSERT(!iterator.Skip()); + return result; + } + + Y_UNIT_TEST_LLVM(TestSortTuples) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listMaker = [&]() + { + TTriple testData[] = { + { 1, 1, 1 }, + { 1, 1, 2 }, + { 1, 2, 3 }, + { 1, 3, 0 }, + { 2, 0, 1 }, + { 2, 1, 0 }, + }; + + TVector<TRuntimeNode> tuplesList; + for (ui32 i = 0; i < Y_ARRAY_SIZE(testData); i++) { + TVector<TRuntimeNode> elements(3); + elements[0] = pb.NewDataLiteral(std::get<0>(testData[i])); + elements[1] = pb.NewDataLiteral(std::get<1>(testData[i])); + elements[2] = pb.NewDataLiteral(std::get<2>(testData[i])); + tuplesList.push_back(pb.NewTuple(elements)); + } + + TVector<TType*> tupleTypes(3); + tupleTypes[0] = pb.NewDataType(NUdf::TDataType<ui32>::Id); + tupleTypes[1] = pb.NewDataType(NUdf::TDataType<ui32>::Id); + tupleTypes[2] = pb.NewDataType(NUdf::TDataType<ui32>::Id); + + return pb.NewList(pb.NewTupleType(tupleTypes), tuplesList); + }; + + { + TRuntimeNode order = TupleOrder(pb, true, true, true); + TTriple expectedData[] = { + { 1, 1, 1 }, + { 1, 1, 2 }, + { 1, 2, 3 }, + { 1, 3, 0 }, + { 2, 0, 1 }, + { 2, 1, 0 }, + }; + TVector<TTriple> expected(expectedData, expectedData + sizeof(expectedData) / sizeof(*expectedData)); + TVector<TTriple> result = SortTuples<LLVM>(setup, listMaker(), order); + UNIT_ASSERT_EQUAL(result, expected); + } + + { + TRuntimeNode order = TupleOrder(pb, false, false, false); + TTriple expectedData[] = { + { 2, 1, 0 }, + { 2, 0, 1 }, + { 1, 3, 0 }, + { 1, 2, 3 }, + { 1, 1, 2 }, + { 1, 1, 1 }, + }; + TVector<TTriple> expected(expectedData, expectedData + sizeof(expectedData) / sizeof(*expectedData)); + TVector<TTriple> result = SortTuples<LLVM>(setup, listMaker(), order); + UNIT_ASSERT_EQUAL(result, expected); + } + + { + TRuntimeNode order = TupleOrder(pb, true, false, true); + TTriple expectedData[] = { + { 1, 3, 0 }, + { 1, 2, 3 }, + { 1, 1, 1 }, + { 1, 1, 2 }, + { 2, 1, 0 }, + { 2, 0, 1 }, + }; + TVector<TTriple> expected(expectedData, expectedData + sizeof(expectedData) / sizeof(*expectedData)); + TVector<TTriple> result = SortTuples<LLVM>(setup, listMaker(), order); + UNIT_ASSERT_EQUAL(result, expected); + } + + { + TRuntimeNode order = TupleOrder(pb, false, true, false); + TTriple expectedData[] = { + { 2, 0, 1 }, + { 2, 1, 0 }, + { 1, 1, 2 }, + { 1, 1, 1 }, + { 1, 2, 3 }, + { 1, 3, 0 }, + }; + TVector<TTriple> expected(expectedData, expectedData + sizeof(expectedData) / sizeof(*expectedData)); + TVector<TTriple> result = SortTuples<LLVM>(setup, listMaker(), order); + UNIT_ASSERT_EQUAL(result, expected); + } + } + + Y_UNIT_TEST_LLVM(TestAsList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto pgmReturn = pb.AsList(pb.NewDataLiteral<ui32>(34)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestListIfTrue) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto pgmReturn = pb.ListIf(pb.NewDataLiteral(true), + pb.NewDataLiteral<ui32>(34)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestListIfFalse) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto pgmReturn = pb.ListIf(pb.NewDataLiteral(false), + pb.NewDataLiteral<ui32>(34)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + UNIT_ASSERT(!iterator.Skip()); + UNIT_ASSERT(!iterator.Skip()); + } + + Y_UNIT_TEST_LLVM(TestNth) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto element0 = pb.NewDataLiteral<ui32>(34); + const auto element1 = pb.NewDataLiteral<ui32>(56); + const auto tuple = pb.NewTuple({element0, element1}); + const auto pgmReturn = pb.Nth(tuple, 1); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto res = graph->GetValue().template Get<ui32>(); + UNIT_ASSERT_VALUES_EQUAL(res, 56); + } + + Y_UNIT_TEST_LLVM(NonDeterministicEnv) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<float>(1); + const auto data2 = pb.NewDataLiteral<float>(2); + const auto data3 = pb.NewDataLiteral<float>(3); + const auto list = pb.NewList(pb.NewDataType(NUdf::TDataType<float>::Id), {data1, data2, data3}); + const auto pgmReturn = pb.Sort(list, + pb.NewDataLiteral(false), + [](TRuntimeNode item) { return item; } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(std::abs(item.template Get<float>() - 3.0) < 0.001); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(std::abs(item.template Get<float>() - 2.0) < 0.001); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(std::abs(item.template Get<float>() - 1.0) < 0.001); + UNIT_ASSERT(false == iterator.Next(item)); + UNIT_ASSERT(false == iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestIndexDictContains) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto item1 = pb.NewDataLiteral<ui32>(7); + const auto item2 = pb.NewDataLiteral<ui32>(16); + const auto list = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), {item1, item2}); + const auto dict = pb.ToIndexDict(list); + const auto type = pb.NewDataType(NUdf::TDataType<ui64>::Id); + const auto key0 = pb.NewDataLiteral<ui64>(0); + const auto key1 = pb.NewDataLiteral<ui64>(1); + const auto key2 = pb.NewDataLiteral<ui64>(2); + const auto keys = pb.NewList(type, {key0, key1, key2}); + const auto pgmReturn = pb.Map(keys, + [&](TRuntimeNode key) { return pb.Contains(dict, key); } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestIndexDictLookup) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto item1 = pb.NewDataLiteral<i32>(7); + const auto item2 = pb.NewDataLiteral<i32>(16); + const auto list = pb.NewList(pb.NewDataType(NUdf::TDataType<i32>::Id), {item1, item2}); + const auto dict = pb.ToIndexDict(list); + const auto type = pb.NewDataType(NUdf::TDataType<ui64>::Id); + const auto key0 = pb.NewDataLiteral<ui64>(0); + const auto key1 = pb.NewDataLiteral<ui64>(1); + const auto key2 = pb.NewDataLiteral<ui64>(2); + const auto keys = pb.NewList(type, {key0, key1, key2}); + const auto pgmReturn = pb.Map(keys, + [&](TRuntimeNode key) { return pb.Lookup(dict, key); } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 16); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestIndexDictContainsLazy) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto item1 = pb.NewDataLiteral<ui32>(7); + const auto item2 = pb.NewDataLiteral<ui32>(16); + const auto list = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), {item1, item2}); + const auto dict = pb.ToIndexDict(pb.LazyList(list)); + const auto type = pb.NewDataType(NUdf::TDataType<ui64>::Id); + const auto key0 = pb.NewDataLiteral<ui64>(0); + const auto key1 = pb.NewDataLiteral<ui64>(1); + const auto key2 = pb.NewDataLiteral<ui64>(2); + const auto keys = pb.NewList(type, {key0, key1, key2}); + const auto pgmReturn = pb.Map(keys, + [&](TRuntimeNode key) { return pb.Contains(dict, key); } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestIndexDictLookupLazy) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto item1 = pb.NewDataLiteral<ui32>(7); + const auto item2 = pb.NewDataLiteral<ui32>(16); + const auto list = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), {item1, item2}); + const auto dict = pb.ToIndexDict(pb.LazyList(list)); + const auto type = pb.NewDataType(NUdf::TDataType<ui64>::Id); + const auto key0 = pb.NewDataLiteral<ui64>(0); + const auto key1 = pb.NewDataLiteral<ui64>(1); + const auto key2 = pb.NewDataLiteral<ui64>(2); + const auto keys = pb.NewList(type, {key0, key1, key2}); + const auto pgmReturn = pb.Map(keys, + [&](TRuntimeNode key) { return pb.Lookup(dict, key); } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 7); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 16); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestToBytes) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto type = pb.NewDataType(NUdf::TDataType<float>::Id); + const auto data1 = pb.NewDataLiteral(0.f); + const auto data2 = pb.NewDataLiteral(-3.14f); + const auto data3 = pb.NewDataLiteral(-HUGE_VALF); + const auto data4 = pb.NewDataLiteral(HUGE_VALF); + const auto list = pb.NewList(type, {data1, data2, data3, data4}); + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.ToBytes(item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "\x00\x00\x00\x00"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "\xc3\xf5\x48\xc0"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "\x00\x00\x80\xff"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "\x00\x00\x80\x7f"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestToBytesOpt) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto type = pb.NewDataType(NUdf::TDataType<float>::Id, true); + const auto data0 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<float>::Id); + const auto data1 = pb.NewOptional(pb.NewDataLiteral(0.f)); + const auto data2 = pb.NewOptional(pb.NewDataLiteral(-3.14f)); + const auto data3 = pb.NewOptional(pb.NewDataLiteral(-HUGE_VALF)); + const auto data4 = pb.NewOptional(pb.NewDataLiteral(HUGE_VALF)); + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4}); + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.ToBytes(item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "\x00\x00\x00\x00"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "\xc3\xf5\x48\xc0"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "\x00\x00\x80\xff"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "\x00\x00\x80\x7f"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestToStringTemporarryUtf8) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto type = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id); + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("long prefix "); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("01234567890 long string"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("01234567890 very long string"); + const auto list = pb.NewList(type, {data1, data2}); + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.ToString(pb.Concat(data0, item)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "long prefix 01234567890 long string"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "long prefix 01234567890 very long string"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFromString) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto type = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("234"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc"); + const auto list = pb.NewList(type, {data1, data2}); + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.FromString(item, pb.NewDataType(NUdf::TDataType<ui32>::Id)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 234); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestStrictFromString) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + TVector<TRuntimeNode> tupleItems; + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("234"); + tupleItems.push_back(pb.StrictFromString(data1, pb.NewDataType(NUdf::TDataType<ui32>::Id))); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-1"); + tupleItems.push_back(pb.StrictFromString(data2, pb.NewDataType(NUdf::TDataType<i32>::Id))); + const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("3.1415926"); + tupleItems.push_back(pb.StrictFromString(data3, pb.NewDataType(NUdf::TDataType<double>::Id))); + const auto pgmReturn = pb.NewTuple(tupleItems); + + { + const auto graph = setup.BuildGraph(pgmReturn); + const auto res = graph->GetValue(); + UNIT_ASSERT_VALUES_EQUAL(res.GetElement(0).template Get<ui32>(), 234); + UNIT_ASSERT_VALUES_EQUAL(res.GetElement(1).template Get<i32>(), -1); + UNIT_ASSERT_VALUES_EQUAL(res.GetElement(2).template Get<double>(), 3.1415926); + } + + { + const auto wrongData = pb.NewDataLiteral<NUdf::EDataSlot::String>("vgfsbhj"); + const auto fail = pb.StrictFromString(wrongData, pb.NewDataType(NUdf::TDataType<ui32>::Id)); + const auto failgraph = setup.BuildGraph(fail, {}); + UNIT_ASSERT_EXCEPTION(failgraph->GetValue(), yexception); + } + + { + const auto wrongData = pb.NewDataLiteral<NUdf::EDataSlot::String>(""); + const auto fail = pb.StrictFromString(wrongData, pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto failgraph = setup.BuildGraph(fail, {}); + UNIT_ASSERT_EXCEPTION(failgraph->GetValue(), yexception); + } + } + + Y_UNIT_TEST_LLVM(TestFromBytes) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + TVector<TRuntimeNode> tupleItems; + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>(TString("\xEA\x00\x00\x00", 4)); + tupleItems.push_back(pb.FromBytes(data1, NUdf::TDataType<ui32>::Id)); + const auto data2 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<const char*>::Id); + tupleItems.push_back(pb.FromBytes(data2, NUdf::TDataType<ui32>::Id)); + const auto pgmReturn = pb.NewTuple(tupleItems); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto res = graph->GetValue(); + UNIT_ASSERT(res.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(res.GetElement(0).template Get<ui32>(), 234); + UNIT_ASSERT(!res.GetElement(1)); + } + + Y_UNIT_TEST_LLVM(TestMTRand) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto seed = pb.NewDataLiteral<ui64>(42); + auto pgmReturn = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui64>::Id)); + auto rnd = pb.NewMTRand(seed); + const ui32 n = 5; + const ui64 expectedValues[n] = { + 13930160852258120406ull, + 11788048577503494824ull, + 13874630024467741450ull, + 2513787319205155662ull, + 16662371453428439381ull, + }; + for (ui32 i = 0; i < n; ++i) { + const auto pair = pb.NextMTRand(rnd); + pgmReturn = pb.Append(pgmReturn, pb.Nth(pair, 0)); + rnd = pb.Nth(pair, 1); + } + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + for (ui32 i = 0; i < n; ++i) { + UNIT_ASSERT(iterator.Next(item)); + const ui64 value = item.template Get<ui64>(); + //Cout << value << Endl; + UNIT_ASSERT_VALUES_EQUAL(value, expectedValues[i]); + } + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestRandom) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const double expectedValue1 = 0.13387664401253274; + const ui64 expectedValue2 = 2516265689700432462; + + const auto rnd1 = pb.Random({}); + const auto rnd2 = pb.RandomNumber({}); + TVector<TRuntimeNode> args; + args.push_back(rnd1); + args.push_back(rnd2); + const auto pgmReturn = pb.NewTuple(args); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto tuple = graph->GetValue(); + UNIT_ASSERT_DOUBLES_EQUAL(tuple.GetElement(0).template Get<double>(), expectedValue1, 1e-3); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).template Get<ui64>(), expectedValue2); + } + + Y_UNIT_TEST_LLVM(TestNow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const ui64 expectedValue = 10000000000000; + + const auto ts = pb.Now({}); + TVector<TRuntimeNode> args; + args.push_back(ts); + const auto pgmReturn = pb.NewTuple(args); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto tuple = graph->GetValue(); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(0).template Get<ui64>(), expectedValue); + } + + Y_UNIT_TEST_LLVM(TestSkipAndTakeOverStream) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto type = pb.NewDataType(NUdf::TDataType<ui8>::Id); + const auto data0 = pb.NewDataLiteral<ui8>(0); + const auto data1 = pb.NewDataLiteral<ui8>(1); + const auto data2 = pb.NewDataLiteral<ui8>(2); + const auto data3 = pb.NewDataLiteral<ui8>(3); + const auto data4 = pb.NewDataLiteral<ui8>(4); + const auto data5 = pb.NewDataLiteral<ui8>(5); + const auto data6 = pb.NewDataLiteral<ui8>(6); + const auto data7 = pb.NewDataLiteral<ui8>(7); + const auto data8 = pb.NewDataLiteral<ui8>(8); + const auto data9 = pb.NewDataLiteral<ui8>(9); + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Take(pb.Skip(pb.Iterator(list, {}), pb.NewDataLiteral<ui64>(4ULL)), pb.NewDataLiteral<ui64>(3ULL)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 5); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 6); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestSkipAndTakeOverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto type = pb.NewDataType(NUdf::TDataType<ui8>::Id); + const auto data0 = pb.NewDataLiteral<ui8>(0); + const auto data1 = pb.NewDataLiteral<ui8>(1); + const auto data2 = pb.NewDataLiteral<ui8>(2); + const auto data3 = pb.NewDataLiteral<ui8>(3); + const auto data4 = pb.NewDataLiteral<ui8>(4); + const auto data5 = pb.NewDataLiteral<ui8>(5); + const auto data6 = pb.NewDataLiteral<ui8>(6); + const auto data7 = pb.NewDataLiteral<ui8>(7); + const auto data8 = pb.NewDataLiteral<ui8>(8); + const auto data9 = pb.NewDataLiteral<ui8>(9); + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.FromFlow(pb.Take(pb.Skip(pb.ToFlow(list), pb.NewDataLiteral<ui64>(4ULL)), pb.NewDataLiteral<ui64>(3ULL))); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 5); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 6); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestLazyListFromArray) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto type = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto data0 = pb.NewDataLiteral<ui32>(1U); + const auto data1 = pb.NewDataLiteral<ui32>(2U); + const auto data2 = pb.NewDataLiteral<ui32>(3U); + const auto array = pb.NewList(type, {data0, data1, data2}); + + const auto pgmReturn = pb.LazyList(array); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto list = graph->GetValue(); + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 3U); + UNIT_ASSERT_VALUES_EQUAL(list.GetElements(), nullptr); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(0).template Get<ui32>(), 1U); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(1).template Get<ui32>(), 2U); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(2).template Get<ui32>(), 3U); + } + + Y_UNIT_TEST_LLVM(TestCollectLazyList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto type = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto data0 = pb.NewDataLiteral<ui32>(1U); + const auto data1 = pb.NewDataLiteral<ui32>(2U); + const auto data2 = pb.NewDataLiteral<ui32>(3U); + const auto array = pb.NewList(type, {data0, data1, data2}); + + const auto pgmReturn = pb.Collect(pb.LazyList(array)); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto list = graph->GetValue(); + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 3U); + UNIT_ASSERT(list.GetElements()); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(0).template Get<ui32>(), 1U); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(1).template Get<ui32>(), 2U); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(2).template Get<ui32>(), 3U); + } + + Y_UNIT_TEST_LLVM(TestAddAllTimezones) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto zones = pb.ListFromRange(pb.NewDataLiteral<ui16>(0U), pb.NewDataLiteral<ui16>(1000U), pb.NewDataLiteral<ui16>(1U)); + const auto pgmReturn = pb.Collect(pb.Map(zones, [&](TRuntimeNode id){ return pb.AddTimezone(pb.CurrentUtcDate({id}), id); })); + const auto graph = setup.BuildGraph(pgmReturn); + + const auto list = graph->GetValue(); + UNIT_ASSERT(list.GetElement(0)); + UNIT_ASSERT(list.GetElement(1)); + UNIT_ASSERT(list.GetElement(2)); + UNIT_ASSERT(list.GetElement(585)); + UNIT_ASSERT(!list.GetElement(586)); + UNIT_ASSERT(list.GetElement(587)); + UNIT_ASSERT(list.GetElement(592)); + UNIT_ASSERT(!list.GetElement(593)); + UNIT_ASSERT(list.GetElement(594)); + } + + Y_UNIT_TEST_LLVM(TestRemoveTimezone) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto type = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>(""); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("2019-10-24T13:01:37,Zulu"); + const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("2019-10-24T13:01:37,Japan"); + const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("2019-10-24T13:01:37,Jamaica"); + + const auto datetimeType = pb.NewDataType(NUdf::EDataSlot::Datetime, true); + const auto datetimeTypeTz = pb.NewDataType(NUdf::EDataSlot::TzDatetime, true); + + const auto list = pb.NewList(type, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.FlatMap(list, + [&](TRuntimeNode data) { + return pb.Convert(pb.FromString(data, datetimeTypeTz), datetimeType); + } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto value = graph->GetValue(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), 3ULL); + + UNIT_ASSERT(!value.GetElement(0U).GetTimezoneId()); + UNIT_ASSERT(!value.GetElement(1U).GetTimezoneId()); + UNIT_ASSERT(!value.GetElement(2U).GetTimezoneId()); + + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(0U).template Get<ui32>(), 1571922097U); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(1U).template Get<ui32>(), 1571889697U); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(2U).template Get<ui32>(), 1571940097U); + } +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 25u + Y_UNIT_TEST_LLVM(TestSqueezeToList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral(0.0); + const auto data1 = pb.NewDataLiteral(1.1); + const auto data2 = pb.NewDataLiteral(-3.14); + const auto data3 = pb.NewDataLiteral(121324.323); + const auto data4 = pb.NewDataLiteral(-7898.8); + const auto type = pb.NewDataType(NUdf::TDataType<double>::Id); + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4}); + + const auto pgmReturn = pb.FromFlow(pb.SqueezeToList(pb.ToFlow(list), pb.NewDataLiteral<ui64>(1000ULL))); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue full; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(full)); + NUdf::TUnboxedValue stub; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(stub)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(stub)); + + UNIT_ASSERT_VALUES_EQUAL(full.GetListLength(), 5ULL); + UNIT_ASSERT_VALUES_EQUAL(full.GetElement(0U).template Get<double>(), 0.0); + UNIT_ASSERT_VALUES_EQUAL(full.GetElement(1U).template Get<double>(), 1.1); + UNIT_ASSERT_VALUES_EQUAL(full.GetElement(2U).template Get<double>(), -3.14); + UNIT_ASSERT_VALUES_EQUAL(full.GetElement(3U).template Get<double>(), 121324.323); + UNIT_ASSERT_VALUES_EQUAL(full.GetElement(4U).template Get<double>(), -7898.8); + } + + Y_UNIT_TEST_LLVM(TestSqueezeToListWithLimit) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral(0.0f); + const auto data1 = pb.NewDataLiteral(1.1f); + const auto data2 = pb.NewDataLiteral(-3.14f); + const auto data3 = pb.NewDataLiteral(12.323f); + const auto data4 = pb.NewDataLiteral(-7898.8f); + const auto type = pb.NewDataType(NUdf::TDataType<float>::Id); + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4}); + + const auto pgmReturn = pb.FromFlow(pb.SqueezeToList(pb.ToFlow(list), pb.NewDataLiteral<ui64>(3ULL))); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue full; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(full)); + NUdf::TUnboxedValue stub; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(stub)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(stub)); + + UNIT_ASSERT_VALUES_EQUAL(full.GetListLength(), 3ULL); + UNIT_ASSERT_VALUES_EQUAL(full.GetElement(0U).template Get<float>(), 0.0f); + UNIT_ASSERT_VALUES_EQUAL(full.GetElement(1U).template Get<float>(), 1.1f); + UNIT_ASSERT_VALUES_EQUAL(full.GetElement(2U).template Get<float>(), -3.14f); + } +#endif + Y_UNIT_TEST_LLVM(TestPerfHolders) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto ui32Type = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto strType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto structType = pb.NewStructType({{"key", ui32Type}, {"value", strType}}); + const auto mark = pb.NewDataLiteral<NUdf::EDataSlot::String>("!"); + + const auto listType = pb.NewListType(structType); + TCallableBuilder listRet(pb.GetTypeEnvironment(), "TestList", listType); + const auto listNode = listRet.Build(); + + const auto pgmReturn = pb.Map(pb.LazyList(TRuntimeNode(listNode, false)), + [&](TRuntimeNode item) { + return pb.NewStruct({ + {"key", pb.Member(item, "key")}, + {"value", pb.AggrConcat(mark, pb.Member(item, "value"))} + }); + }); + + std::vector<ui32> src(10000U); + std::iota(src.begin(), src.end(), 0U); + + const auto myStructFactory = [](const THolderFactory& factory, ui32 i) { + NUdf::TUnboxedValue* itemsPtr = nullptr; + const auto structObj = factory.CreateDirectArrayHolder(2U, itemsPtr); + itemsPtr[0] = NUdf::TUnboxedValuePod(ui32(i)); + itemsPtr[1] = MakeString("ABCDEFGHIJKL"); + return structObj; + }; + + const auto t1 = TInstant::Now(); + { + const auto graph = setup.BuildGraph(pgmReturn, {listNode}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(src.size(), items)); + std::transform(src.cbegin(), src.cend(), items, std::bind(myStructFactory, std::ref(graph->GetHolderFactory()), std::placeholders::_1)); + + const auto iterator = graph->GetValue().GetListIterator(); + ui32 i = 0U; + for (NUdf::TUnboxedValue current; iterator.Next(current); ++i) { + UNIT_ASSERT_VALUES_EQUAL(current.GetElement(0).template Get<ui32>(), i); + UNBOXED_VALUE_STR_EQUAL(current.GetElement(1), "!ABCDEFGHIJKL"); + } + } + + const auto t2 = TInstant::Now(); + Cout << t2 - t1 << Endl; + } + + Y_UNIT_TEST_LLVM(TestPerfGrep) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto ui32Type = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto strType = pb.NewDataType(NUdf::TDataType<char*>::Id); + auto structType = pb.NewEmptyStructType(); + structType = pb.NewStructType(structType, "key", ui32Type); + structType = pb.NewStructType(structType, "value", strType); + const auto keyIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("key"); + const auto valueIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("value"); + + TRuntimeNode rowArg = pb.Arg(structType); + + const auto pgmReturn = pb.Equals(pb.Member(rowArg, "value"), + pb.NewDataLiteral<NUdf::EDataSlot::String>("ABCDE")); + + const auto t1 = TInstant::Now(); + { + const auto graph = setup.BuildGraph(pgmReturn, {rowArg.GetNode()}); + const auto row = graph->GetEntryPoint(0, true); + + NUdf::TUnboxedValue* items = nullptr; + const auto structObj = graph->GetHolderFactory().CreateDirectArrayHolder(2, items); + graph->Prepare(); + const ui32 n = 10000; + for (ui32 i = 0; i < n; ++i) { + items[keyIndex] = NUdf::TUnboxedValuePod(i); + items[valueIndex] = NUdf::TUnboxedValuePod::Embedded("ABCDF"); + row->SetValue(graph->GetContext(), NUdf::TUnboxedValuePod(structObj)); + const bool keep = graph->GetValue().template Get<bool>(); + UNIT_ASSERT(!keep); + } + } + + const auto t2 = TInstant::Now(); + Cout << t2 - t1 << Endl; + } + + Y_NO_INLINE NUdf::TUnboxedValuePod SpecialFunc(const NUdf::TUnboxedValuePod* args) { + const auto stringRef = args[1].AsStringRef(); + return NUdf::TUnboxedValuePod(stringRef.Size() == 5 && std::memcmp(stringRef.Data(), "ABCDE", 5) == 0); + } + + Y_UNIT_TEST_LLVM(TestPerfGrepSpecialFunc) { + const auto t1 = TInstant::Now(); + { + NUdf::TUnboxedValuePod items[2]; + const ui32 n = 10000; + for (ui32 i = 0; i < n; ++i) { + items[0] = NUdf::TUnboxedValuePod(i); + items[1] = NUdf::TUnboxedValuePod::Embedded("ABCDF"); + bool keep = SpecialFunc(items).template Get<bool>(); + UNIT_ASSERT(!keep); + } + } + + const auto t2 = TInstant::Now(); + Cout << t2 - t1 << Endl; + } +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_condense_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_condense_ut.cpp new file mode 100644 index 00000000000..653d6c0cc08 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_condense_ut.cpp @@ -0,0 +1,496 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> + +namespace NKikimr { +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLCondenseNodeTest) { + Y_UNIT_TEST_LLVM(TestSqueeze) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto data1 = pb.NewOptional(pb.NewDataLiteral<double>(3.8)); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<double>(-53.2)); + const auto data3 = pb.NewOptional(pb.NewDataLiteral<double>(233.8)); + const auto data4 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<double>::Id); + const auto data0 = pb.NewOptional(pb.NewDataLiteral<double>(HUGE_VAL)); + const auto list = pb.NewList(dataType, {data4, data3, data2, data1}); + + const auto pgmReturn = pb.Squeeze(pb.Iterator(list, {}), data0, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.AggrMin(item, state); + } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -53.2); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestSqueezeOnEmpty) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto data0 = pb.NewOptional(pb.NewDataLiteral<double>(HUGE_VAL)); + const auto list = pb.NewEmptyList(dataType); + + const auto pgmReturn = pb.Squeeze(pb.Iterator(list, {}), data0, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.AggrMin(item, state); + } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), HUGE_VAL); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestSqueeze1OverEmpty) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id); + const auto list = pb.NewEmptyList(dataType); + const auto pgmReturn = pb.Squeeze1(pb.Iterator(list, {}), + [&](TRuntimeNode item) { + return pb.Minus(item); + }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.Mul(item, state); + } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestSqueeze1OverSingle) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id); + const auto data1 = pb.NewDataLiteral<i32>(1); + const auto list = pb.NewList(dataType, {data1}); + const auto pgmReturn = pb.Squeeze1(pb.Iterator(list, {}), + [&](TRuntimeNode item) { + return pb.Minus(item); + }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.Mul(item, state); + } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1); + + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestSqueeze1OverMany) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id); + const auto data1 = pb.NewDataLiteral<i32>(1); + const auto data2 = pb.NewDataLiteral<i32>(2); + const auto data3 = pb.NewDataLiteral<i32>(7); + const auto list = pb.NewList(dataType, {data1, data2, data3}); + const auto pgmReturn = pb.Squeeze1(pb.Iterator(list, {}), + [&](TRuntimeNode item) { + return pb.Minus(item); + }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.Mul(item, state); + } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -14); + + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestCondense) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto data0 = pb.NewOptional(pb.NewDataLiteral<double>(0.0)); + const auto data1 = pb.NewOptional(pb.NewDataLiteral<double>(3.8)); + const auto data2 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<double>::Id); + const auto data3 = pb.NewOptional(pb.NewDataLiteral<double>(-53.2)); + const auto data4 = pb.NewOptional(pb.NewDataLiteral<double>(233.8)); + const auto data5 = pb.NewOptional(pb.NewDataLiteral<double>(3.14)); + const auto data6 = pb.NewOptional(pb.NewDataLiteral<double>(-73.12)); + const auto data7 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<double>::Id); + const auto data8 = pb.NewOptional(pb.NewDataLiteral<double>(233.8)); + const auto data9 = pb.NewOptional(pb.NewDataLiteral<double>(1221.8)); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Condense(pb.Iterator(list, {}), data0, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.Or({pb.Not(pb.Exists(item)), pb.Less(state, data0)}); + }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.AggrAdd(item, state); + } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 3.8); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -53.2); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 163.82); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 1455.6); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestCondense1) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id); + const auto data0 = pb.NewDataLiteral<i32>(-1); + const auto data1 = pb.NewDataLiteral<i32>(2); + const auto data2 = pb.NewDataLiteral<i32>(0); + const auto data3 = pb.NewDataLiteral<i32>(7); + const auto data4 = pb.NewDataLiteral<i32>(5); + const auto data5 = pb.NewDataLiteral<i32>(-7); + const auto data6 = pb.NewDataLiteral<i32>(-6); + const auto data7 = pb.NewDataLiteral<i32>(4); + const auto data8 = pb.NewDataLiteral<i32>(8); + const auto data9 = pb.NewDataLiteral<i32>(9); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + const auto pgmReturn = pb.Condense1(pb.Iterator(list, {}), + [&](TRuntimeNode item) { + return pb.Minus(item); + }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.LessOrEqual(item, state); + }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.Mul(item, state); + } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 2); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 0); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 6); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -288); + + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestCondenseOverList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto data0 = pb.NewOptional(pb.NewDataLiteral<double>(0.0)); + const auto data1 = pb.NewOptional(pb.NewDataLiteral<double>(3.8)); + const auto data2 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<double>::Id); + const auto data3 = pb.NewOptional(pb.NewDataLiteral<double>(-53.2)); + const auto data4 = pb.NewOptional(pb.NewDataLiteral<double>(233.8)); + const auto data5 = pb.NewOptional(pb.NewDataLiteral<double>(3.14)); + const auto data6 = pb.NewOptional(pb.NewDataLiteral<double>(-73.12)); + const auto data7 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<double>::Id); + const auto data8 = pb.NewOptional(pb.NewDataLiteral<double>(233.8)); + const auto data9 = pb.NewOptional(pb.NewDataLiteral<double>(1221.8)); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Condense(list, data0, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.Or({pb.Not(pb.Exists(item)), pb.Less(state, data0)}); + }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.AggrAdd(item, state); + } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 3.8); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -53.2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 163.82); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 1455.6); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestCondense1OverList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id); + const auto data0 = pb.NewDataLiteral<i32>(-1); + const auto data1 = pb.NewDataLiteral<i32>(2); + const auto data2 = pb.NewDataLiteral<i32>(0); + const auto data3 = pb.NewDataLiteral<i32>(7); + const auto data4 = pb.NewDataLiteral<i32>(5); + const auto data5 = pb.NewDataLiteral<i32>(-7); + const auto data6 = pb.NewDataLiteral<i32>(-6); + const auto data7 = pb.NewDataLiteral<i32>(4); + const auto data8 = pb.NewDataLiteral<i32>(8); + const auto data9 = pb.NewDataLiteral<i32>(9); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + const auto pgmReturn = pb.Condense1(list, + [&](TRuntimeNode item) { + return pb.Minus(item); + }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.LessOrEqual(item, state); + }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.Mul(item, state); + } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 6); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -288); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestCondenseInterrupt) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<bool>::Id)); + const auto data0 = pb.NewOptional(pb.NewDataLiteral<bool>(false)); + const auto data1 = pb.NewOptional(pb.NewDataLiteral<bool>(false)); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<bool>(false)); + const auto data3 = pb.NewOptional(pb.NewDataLiteral<bool>(false)); + const auto data4 = pb.NewOptional(pb.NewDataLiteral<bool>(true)); + const auto data5 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id); + const auto data6 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6}); + + const auto pgmReturn = pb.FromFlow(pb.Condense(pb.ToFlow(list), pb.NewDataLiteral<bool>(false), + [&](TRuntimeNode, TRuntimeNode state) { + return pb.If(state, pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id), pb.NewDataLiteral<bool>(false)); + }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.Or({pb.Unwrap(item, pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0), state}); + } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(item.template Get<bool>()); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestCondense1Interrupt) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<bool>::Id)); + const auto data0 = pb.NewOptional(pb.NewDataLiteral<bool>(true)); + const auto data1 = pb.NewOptional(pb.NewDataLiteral<bool>(true)); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<bool>(true)); + const auto data3 = pb.NewOptional(pb.NewDataLiteral<bool>(true)); + const auto data4 = pb.NewOptional(pb.NewDataLiteral<bool>(false)); + const auto data5 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id); + const auto data6 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6}); + + const auto pgmReturn = pb.FromFlow(pb.Condense1(pb.ToFlow(list), + [&](TRuntimeNode item) { + return pb.Unwrap(item, pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0); + }, + [&](TRuntimeNode, TRuntimeNode state) { + return pb.If(state, pb.NewDataLiteral<bool>(false), pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id)); + }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.And({pb.Unwrap(item, pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0), state}); + } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item.template Get<bool>()); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestCondenseInterruptEndlessStream) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto pgmReturn = pb.Condense(pb.SourceOf(pb.NewStreamType(pb.NewNull().GetStaticType())), pb.NewDataLiteral<ui32>(0U), + [&](TRuntimeNode, TRuntimeNode state) { + return pb.If(pb.AggrLess(state, pb.NewDataLiteral<ui32>(123456U)), pb.NewDataLiteral<bool>(false), pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id)); + }, + [&](TRuntimeNode, TRuntimeNode state) { + return pb.AggrAdd(pb.NewDataLiteral<ui32>(1U), state); + } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 123456U); + + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestCondense1InterruptEndlessFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto pgmReturn = pb.FromFlow(pb.Condense1(pb.SourceOf(pb.NewFlowType(pb.NewNull().GetStaticType())), + [&](TRuntimeNode) { + return pb.NewDataLiteral<ui32>(0U); + }, + [&](TRuntimeNode, TRuntimeNode state) { + return pb.If(pb.AggrLess(state, pb.NewDataLiteral<ui32>(123456U)), pb.NewDataLiteral<bool>(false), pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id)); + }, + [&](TRuntimeNode, TRuntimeNode state) { + return pb.AggrAdd(pb.NewDataLiteral<ui32>(1U), state); + } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 123456U); + + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + + Y_UNIT_TEST_LLVM(TestCondenseListeralListInMap) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("other"); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("foo"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("bar"); + const auto type = pb.NewDataType(NUdf::EDataSlot::String); + const auto list2 = pb.NewList(type, {data1, data2}); + const auto list0 = pb.NewList(type, {data1, data2, data0}); + + const auto pgmReturn = pb.Map(list0, + [&](TRuntimeNode item) { + return pb.Head(pb.Condense(list2, + pb.NewDataLiteral(false), + [&](TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral(false); }, + [&](TRuntimeNode it, TRuntimeNode state) { return pb.Or({state, pb.AggrEquals(item, it)}); } + )); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.Get<bool>()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.Get<bool>()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.Get<bool>()); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestCondense1ListeralListInMap) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("other"); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("foo"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("bar"); + const auto type = pb.NewDataType(NUdf::EDataSlot::String); + const auto list2 = pb.NewList(type, {data1, data2}); + const auto list0 = pb.NewList(type, {data1, data2, data0}); + + const auto pgmReturn = pb.Map(list0, + [&](TRuntimeNode item) { + return pb.Head(pb.Condense1(list2, + [&](TRuntimeNode it) { return pb.AggrEquals(item, it); }, + [&](TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral(false); }, + [&](TRuntimeNode it, TRuntimeNode state) { return pb.Or({state, pb.AggrEquals(item, it)}); } + )); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.Get<bool>()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.Get<bool>()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.Get<bool>()); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_decimal_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_decimal_ut.cpp new file mode 100644 index 00000000000..00fa7eca1b6 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_decimal_ut.cpp @@ -0,0 +1,2329 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> + +namespace NKikimr { +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLDecimalTest) { + Y_UNIT_TEST_LLVM(TestNanvl) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto type = pb.NewOptionalType(pb.NewDecimalType(13, 5)); + const auto data0 = pb.NewOptional(pb.NewDecimalLiteral(0, 13, 5)); + const auto data1 = pb.NewOptional(pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 13, 5)); + const auto data2 = pb.NewOptional(pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 13, 5)); + const auto data3 = pb.NewOptional(pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 13, 5)); + const auto data4 = pb.NewEmptyOptional(type); + const auto data5 = pb.NewOptional(pb.NewDecimalLiteral(-NYql::NDecimal::Nan(), 13, 5)); + const auto data = pb.NewDecimalLiteral(314159, 13, 5); + + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Nanvl(item, data); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 314159); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == +NYql::NDecimal::Inf()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 314159); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestToIntegral) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto type = pb.NewDecimalType(13, 1); + const auto data0 = pb.NewDecimalLiteral(0, 13, 1); + const auto data1 = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 13, 1); + const auto data2 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 13, 1); + const auto data3 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 13, 1); + const auto data4 = pb.NewDecimalLiteral(1270, 13, 1); + const auto data5 = pb.NewDecimalLiteral(-1280, 13, 1); + const auto data6 = pb.NewDecimalLiteral(2550, 13, 1); + const auto data7 = pb.NewDecimalLiteral(-2560, 13, 1); + const auto data8 = pb.NewDecimalLiteral(2560, 13, 1); + const auto data9 = pb.NewDecimalLiteral(-2570, 13, 1); + const auto dataA = pb.NewDecimalLiteral(327670, 13, 1); + const auto dataB = pb.NewDecimalLiteral(-327680, 13, 1); + const auto dataC = pb.NewDecimalLiteral(655350, 13, 1); + const auto dataD = pb.NewDecimalLiteral(-655360, 13, 1); + const auto dataE = pb.NewDecimalLiteral(21474836470, 13, 1); + const auto dataF = pb.NewDecimalLiteral(-21474836480, 13, 1); + const auto dataG = pb.NewDecimalLiteral(21474836480, 13, 1); + const auto dataH = pb.NewDecimalLiteral(-21474836490, 13, 1); + + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9, dataA, dataB, dataC, dataD, dataE, dataF, dataG, dataH}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.NewTuple({ + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i8>::Id, true)), + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui8>::Id, true)), + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i16>::Id, true)), + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui16>::Id, true)), + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i32>::Id, true)), + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui32>::Id, true)), + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i64>::Id, true)), + pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui64>::Id, true)) + }); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[1].Get<ui8>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[7].Get<ui64>(), 0); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT(!item.GetElements()[2]); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT(!item.GetElements()[4]); + UNIT_ASSERT(!item.GetElements()[5]); + UNIT_ASSERT(!item.GetElements()[6]); + UNIT_ASSERT(!item.GetElements()[7]); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT(!item.GetElements()[2]); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT(!item.GetElements()[4]); + UNIT_ASSERT(!item.GetElements()[5]); + UNIT_ASSERT(!item.GetElements()[6]); + UNIT_ASSERT(!item.GetElements()[7]); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT(!item.GetElements()[2]); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT(!item.GetElements()[4]); + UNIT_ASSERT(!item.GetElements()[5]); + UNIT_ASSERT(!item.GetElements()[6]); + UNIT_ASSERT(!item.GetElements()[7]); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), 127); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[1].Get<ui8>(), 127); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), 127); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), 127); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 127); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 127); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 127); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[7].Get<ui64>(), 127); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), -128); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), -128); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), -128); + UNIT_ASSERT(!item.GetElements()[5]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), -128); + UNIT_ASSERT(!item.GetElements()[7]); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[1].Get<ui8>(), 255); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), 255); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), 255); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 255); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 255); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 255); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[7].Get<ui64>(), 255); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), -256); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), -256); + UNIT_ASSERT(!item.GetElements()[5]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), -256); + UNIT_ASSERT(!item.GetElements()[7]); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), 256); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), 256); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 256); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 256); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 256); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[7].Get<ui64>(), 256); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), -257); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), -257); + UNIT_ASSERT(!item.GetElements()[5]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), -257); + UNIT_ASSERT(!item.GetElements()[7]); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), 32767); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), 32767); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 32767); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 32767); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 32767); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[7].Get<ui64>(), 32767); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), -32768); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), -32768); + UNIT_ASSERT(!item.GetElements()[5]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), -32768); + UNIT_ASSERT(!item.GetElements()[7]); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT(!item.GetElements()[2]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), 65535); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 65535); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 65535); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 65535); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[7].Get<ui64>(), 65535); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT(!item.GetElements()[2]); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), -65536); + UNIT_ASSERT(!item.GetElements()[5]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), -65536); + UNIT_ASSERT(!item.GetElements()[7]); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT(!item.GetElements()[2]); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 2147483647LL); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 2147483647LL); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 2147483647LL); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[7].Get<ui64>(), 2147483647LL); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT(!item.GetElements()[2]); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), -2147483648LL); + UNIT_ASSERT(!item.GetElements()[5]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), -2147483648LL); + UNIT_ASSERT(!item.GetElements()[7]); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT(!item.GetElements()[2]); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT(!item.GetElements()[4]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 2147483648LL); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 2147483648LL); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[7].Get<ui64>(), 2147483648LL); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElements()[0]); + UNIT_ASSERT(!item.GetElements()[1]); + UNIT_ASSERT(!item.GetElements()[2]); + UNIT_ASSERT(!item.GetElements()[3]); + UNIT_ASSERT(!item.GetElements()[4]); + UNIT_ASSERT(!item.GetElements()[5]); + UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), -2147483649LL); + UNIT_ASSERT(!item.GetElements()[7]); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestToFloat) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDecimalType(10, 3); + const auto data0 = pb.NewDecimalLiteral(2123, 10, 3); + const auto data1 = pb.NewDecimalLiteral(233, 10, 3); + const auto data2 = pb.NewDecimalLiteral(0, 10, 3); + const auto data3 = pb.NewDecimalLiteral(-3277823, 10, 3); + const auto data4 = pb.NewDecimalLiteral(-1, 10, 3); + const auto data5 = pb.NewDecimalLiteral(7128, 10, 3); + const auto data6 = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 10, 3); + const auto data7 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 10, 3); + const auto data8 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 10, 3); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Convert(item, pb.NewDataType(NUdf::TDataType<float>::Id)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 2.123f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 0.233f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 0.0f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -3277.823f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -0.001f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 7.128f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(std::isnan(item.template Get<float>())); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(std::isinf(item.template Get<float>()) && item.template Get<float>() > 0.0f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(std::isinf(item.template Get<float>()) && item.template Get<float>() < 0.0f); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestToDouble) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDecimalType(10, 5); + const auto data0 = pb.NewDecimalLiteral(2123, 10, 5); + const auto data1 = pb.NewDecimalLiteral(233, 10, 5); + const auto data2 = pb.NewDecimalLiteral(0, 10, 5); + const auto data3 = pb.NewDecimalLiteral(-3277823, 10, 5); + const auto data4 = pb.NewDecimalLiteral(-1, 10, 5); + const auto data5 = pb.NewDecimalLiteral(7128, 10, 5); + const auto data6 = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 10, 5); + const auto data7 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 10, 5); + const auto data8 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 10, 5); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Convert(item, pb.NewDataType(NUdf::TDataType<double>::Id)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.02123); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.00233); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -32.77823); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -0.00001); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.07128); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(std::isnan(item.template Get<double>())); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(std::isinf(item.template Get<double>()) && item.template Get<double>() > 0.0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(std::isinf(item.template Get<double>()) && item.template Get<double>() < 0.0); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestDiv) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDecimalType(10, 0); + const auto data0 = pb.NewDecimalLiteral(2, 10, 0); + const auto data1 = pb.NewDecimalLiteral(23, 10, 0); + const auto data2 = pb.NewDecimalLiteral(-23, 10, 0); + const auto data3 = pb.NewDecimalLiteral(25, 10, 0); + const auto data4 = pb.NewDecimalLiteral(-25, 10, 0); + const auto data5 = pb.NewDecimalLiteral(1, 10, 0); + const auto data6 = pb.NewDecimalLiteral(-1, 10, 0); + const auto data7 = pb.NewDecimalLiteral(3, 10, 0); + const auto data8 = pb.NewDecimalLiteral(-3, 10, 0); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.DecimalDiv(item, data0); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 12); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -12); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 12); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -12); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -2); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestDivInt) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<i8>::Id); + const auto data0 = pb.NewDecimalLiteral(-238973, 9, 3); + const auto data1 = pb.NewDataLiteral<i8>(0); + const auto data2 = pb.NewDataLiteral<i8>(-1); + const auto data3 = pb.NewDataLiteral<i8>(-128); + const auto data4 = pb.NewDataLiteral<i8>(3); + const auto data5 = pb.NewDataLiteral<i8>(5); + const auto data6 = pb.NewDataLiteral<i8>(-7); + const auto data7 = pb.NewDataLiteral<i8>(13); + const auto data8 = pb.NewDataLiteral<i8>(-19); + const auto data9 = pb.NewDataLiteral<i8>(42); + const auto list = pb.NewList(dataType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.DecimalDiv(data0, item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 238973); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 1866); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -79658); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -47795); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 34139); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -18383); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 12577); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -5690); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestMod) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDecimalType(5, 2); + const auto data0 = pb.NewDecimalLiteral(-12323, 5, 2); + const auto data1 = pb.NewDecimalLiteral(0, 5, 2); + const auto data2 = pb.NewDecimalLiteral(NYql::NDecimal::Inf(), 5, 2); + const auto data3 = pb.NewDecimalLiteral(-1, 5, 2); + const auto data4 = pb.NewDecimalLiteral(2, 5, 2); + const auto data5 = pb.NewDecimalLiteral(-3, 5, 2); + const auto data6 = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 5, 2); + const auto data7 = pb.NewDecimalLiteral(7, 5, 2); + const auto data8 = pb.NewDecimalLiteral(-10000, 5, 2); + const auto data9 = pb.NewDecimalLiteral(12329, 5, 2); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.DecimalMod(data0, item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -2323); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -12323); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestModInt) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<i16>::Id); + const auto data0 = pb.NewDecimalLiteral(-743, 3, 2); + const auto data1 = pb.NewDataLiteral<i16>(0); + const auto data2 = pb.NewDataLiteral<i16>(1); + const auto data3 = pb.NewDataLiteral<i16>(-2); + const auto data4 = pb.NewDataLiteral<i16>(3); + const auto data5 = pb.NewDataLiteral<i16>(4); + const auto data6 = pb.NewDataLiteral<i16>(-5); + const auto data7 = pb.NewDataLiteral<i16>(8); + const auto data8 = pb.NewDataLiteral<i16>(10); + const auto data9 = pb.NewDataLiteral<i16>(-10); + const auto list = pb.NewList(dataType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.DecimalMod(data0, item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -43); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -143); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -143); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -343); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -243); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -743); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -743); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -743); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestMul) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDecimalType(10, 2); + const auto data0 = pb.NewDecimalLiteral(333, 10, 2); + const auto data1 = pb.NewDecimalLiteral(-100, 10, 2); + const auto data2 = pb.NewDecimalLiteral(-120, 10, 2); + const auto data3 = pb.NewDecimalLiteral(3, 10, 2); + const auto data4 = pb.NewDecimalLiteral(77, 10, 2); + const auto data5 = pb.NewDecimalLiteral(122, 10, 2); + const auto data6 = pb.NewDecimalLiteral(1223, 10, 2); + const auto data7 = pb.NewDecimalLiteral(-999, 10, 2); + const auto data8 = pb.NewDecimalLiteral(0, 10, 2); + const auto data9 = pb.NewDecimalLiteral(-3003003003LL, 10, 2); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.DecimalMul(item, data0); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 1109); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -333); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -400); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 10); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 256); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 406); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 4073); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -3327); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestMulUInt) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id); + const auto data0 = pb.NewDecimalLiteral(-333, 7, 2); + const auto data1 = pb.NewDataLiteral<ui16>(0); + const auto data2 = pb.NewDataLiteral<ui16>(1); + const auto data3 = pb.NewDataLiteral<ui16>(2); + const auto data4 = pb.NewDataLiteral<ui16>(3); + const auto data5 = pb.NewDataLiteral<ui16>(10); + const auto data6 = pb.NewDataLiteral<ui16>(100); + const auto data7 = pb.NewDataLiteral<ui16>(1000); + const auto data8 = pb.NewDataLiteral<ui16>(10000); + const auto data9 = pb.NewDataLiteral<ui16>(65535); + const auto list = pb.NewList(dataType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.DecimalMul(data0, item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -333); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -666); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -999); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -3330); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -33300); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -333000); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -3330000); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestMulTinyInt) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<i8>::Id); + const auto data0 = pb.NewDecimalLiteral(3631400, 32, 4); + const auto data1 = pb.NewDataLiteral<i8>(0); + const auto data2 = pb.NewDataLiteral<i8>(1); + const auto data3 = pb.NewDataLiteral<i8>(-1); + const auto data4 = pb.NewDataLiteral<i8>(3); + const auto data5 = pb.NewDataLiteral<i8>(-3); + const auto data6 = pb.NewDataLiteral<i8>(100); + const auto data7 = pb.NewDataLiteral<i8>(-100); + const auto data8 = pb.NewDataLiteral<i8>(127); + const auto data9 = pb.NewDataLiteral<i8>(-128); + const auto list = pb.NewList(dataType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.DecimalMul(data0, item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 3631400); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -3631400); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 10894200); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -10894200); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 363140000); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -363140000); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 461187800); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -464819200); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestCastAndMulTinyInt) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDecimalType(32, 4); + const auto data0 = pb.NewDataLiteral<i8>(1); + const auto data1 = pb.NewDecimalLiteral(3145926, 32, 4); + const auto data2 = pb.NewDecimalLiteral(-3145926, 32, 4); + const auto list = pb.NewList(dataType, {data1, data2}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.NewTuple({pb.DecimalMul(item, data0), pb.DecimalMul(item, pb.ToDecimal(data0, 32, 4))}); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == 3145926); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 3145926); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -3145926); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -3145926); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestLongintMul) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDecimalType(10, 0); + const auto data0 = pb.NewDecimalLiteral(333, 10, 0); + const auto data1 = pb.NewDecimalLiteral(-100, 10, 0); + const auto data2 = pb.NewDecimalLiteral(-120, 10, 0); + const auto data3 = pb.NewDecimalLiteral(3, 10, 0); + const auto data4 = pb.NewDecimalLiteral(77, 10, 0); + const auto data5 = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 10, 0); + const auto data6 = pb.NewDecimalLiteral(30030031, 10, 0); + const auto data7 = pb.NewDecimalLiteral(-30030031, 10, 0); + const auto data8 = pb.NewDecimalLiteral(0, 10, 0); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.DecimalMul(item, data0); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 110889); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -33300); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -39960); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 999); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 25641); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == NYql::NDecimal::Inf()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 0); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestScaleUp) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDecimalType(10, 2); + const auto data0 = pb.NewDecimalLiteral(333, 10, 2); + const auto data1 = pb.NewDecimalLiteral(-100, 10, 2); + const auto data2 = pb.NewDecimalLiteral(-120, 10, 2); + const auto data3 = pb.NewDecimalLiteral(3, 10, 2); + const auto data4 = pb.NewDecimalLiteral(77, 10, 2); + const auto data5 = pb.NewDecimalLiteral(122, 10, 2); + const auto data6 = pb.NewDecimalLiteral(1223, 10, 2); + const auto data7 = pb.NewDecimalLiteral(-999, 10, 2); + const auto data8 = pb.NewDecimalLiteral(0, 10, 2); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.ToDecimal(item, 12, 4); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 33300); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -10000); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -12000); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 300); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 7700); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 12200); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 122300); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -99900); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 0); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestScaleDown) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDecimalType(10, 2); + const auto data0 = pb.NewDecimalLiteral(-251, 10, 2); + const auto data1 = pb.NewDecimalLiteral(-250, 10, 2); + const auto data2 = pb.NewDecimalLiteral(-150, 10, 2); + const auto data3 = pb.NewDecimalLiteral(-51, 10, 2); + const auto data4 = pb.NewDecimalLiteral(50, 10, 2); + const auto data5 = pb.NewDecimalLiteral(50, 10, 2); + const auto data6 = pb.NewDecimalLiteral(51, 10, 2); + const auto data7 = pb.NewDecimalLiteral(150, 10, 2); + const auto data8 = pb.NewDecimalLiteral(250, 10, 2); + const auto data9 = pb.NewDecimalLiteral(251, 10, 2); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.ToDecimal(item, 8, 0); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 3); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestMinMax) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDecimalLiteral(-NYql::NDecimal::Nan(), 13, 2); + const auto data2 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 13, 2); + const auto data3 = pb.NewDecimalLiteral(314, 13, 2); + const auto data4 = pb.NewDecimalLiteral(-213, 13, 2); + const auto data5 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 13, 2); + const auto dataType = pb.NewDecimalType(13, 2); + const auto list = pb.NewList(dataType, {data1, data2, data3, data4, data5}); + const auto pgmReturn = pb.FlatMap(list, + [&](TRuntimeNode left) { + return pb.Map(list, + [&](TRuntimeNode right) { + return pb.NewTuple({pb.Min(left, right), pb.Max(left, right)}); + }); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Nan()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == +NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == 314); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 314); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -213); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -213); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == 314); + UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -213); + UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == 314); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 314); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == 314); + UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == 314); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 314); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -213); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 314); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 314); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -213); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -213); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -213); + UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -213); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 314); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -213); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -213); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -213); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 314); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -213); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf()); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestAggrMinMax) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 13, 2); + const auto data2 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 13, 2); + const auto data3 = pb.NewDecimalLiteral(314, 13, 2); + const auto data4 = pb.NewDecimalLiteral(-213, 13, 2); + const auto data5 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 13, 2); + const auto dataType = pb.NewDecimalType(13, 2); + const auto list = pb.NewList(dataType, {data1, data2, data3, data4, data5}); + const auto pgmReturn = pb.FlatMap(list, + [&](TRuntimeNode left) { + return pb.Map(list, + [&](TRuntimeNode right) { + return pb.NewTuple({pb.AggrMin(left, right), pb.AggrMax(left, right)}); + }); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == +NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == 314); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -213); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == 314); + UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -213); + UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == 314); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == 314); + UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == 314); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 314); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -213); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 314); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 314); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -213); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -213); + UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -213); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 314); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -213); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -213); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -213); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 314); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -213); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf()); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestAddSub) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDecimalLiteral(-NYql::NDecimal::Nan(), 13, 2); + const auto data2 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 13, 2); + const auto data3 = pb.NewDecimalLiteral(314, 13, 2); + const auto data4 = pb.NewDecimalLiteral(-213, 13, 2); + const auto data5 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 13, 2); + + const auto dataType = pb.NewDecimalType(13, 2); + const auto list = pb.NewList(dataType, {data1, data2, data3, data4, data5}); + const auto pgmReturn = pb.FlatMap(list, + [&](TRuntimeNode left) { + return pb.Map(list, + [&](TRuntimeNode right) { + return pb.NewTuple({pb.Add(left, right), pb.Sub(left, right)}); + }); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == +NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == 628); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 0); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == 101); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 527); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == +NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == 101); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -527); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -426); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 0); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestCompares) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1l = pb.NewDecimalLiteral(-7, 10, 0); + const auto data2l = pb.NewDecimalLiteral(3, 10, 0); + const auto data3l = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 10, 0); + const auto data4l = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 10, 0); + const auto data5l = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 10, 0); + + const auto data1r = pb.NewDecimalLiteral(-700, 7, 2); + const auto data2r = pb.NewDecimalLiteral(300, 7, 2); + const auto data3r = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 7, 2); + const auto data4r = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 7, 2); + const auto data5r = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 7, 2); + + auto pairType = pb.NewTupleType({pb.NewDecimalType(10, 0), pb.NewDecimalType(7, 2)}); + const auto list = pb.NewList(pairType, { + pb.NewTuple({data1l, data1r}), + pb.NewTuple({data1l, data2r}), + pb.NewTuple({data1l, data3r}), + pb.NewTuple({data1l, data4r}), + pb.NewTuple({data1l, data5r}), + + pb.NewTuple({data2l, data1r}), + pb.NewTuple({data2l, data2r}), + pb.NewTuple({data2l, data3r}), + pb.NewTuple({data2l, data4r}), + pb.NewTuple({data2l, data5r}), + + pb.NewTuple({data3l, data1r}), + pb.NewTuple({data3l, data2r}), + pb.NewTuple({data3l, data3r}), + pb.NewTuple({data3l, data4r}), + pb.NewTuple({data3l, data5r}), + + pb.NewTuple({data4l, data1r}), + pb.NewTuple({data4l, data2r}), + pb.NewTuple({data4l, data3r}), + pb.NewTuple({data4l, data4r}), + pb.NewTuple({data4l, data5r}), + + pb.NewTuple({data5l, data1r}), + pb.NewTuple({data5l, data2r}), + pb.NewTuple({data5l, data3r}), + pb.NewTuple({data5l, data4r}), + pb.NewTuple({data5l, data5r}), + }); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.NewTuple({ + pb.Equals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.NotEquals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.Less(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.LessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.Greater(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.GreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)) + }); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestComparesWithIntegral) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto optType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i64>::Id)); + + const auto data1l = pb.NewOptional(pb.NewDataLiteral<i64>(-7LL)); + const auto data2l = pb.NewOptional(pb.NewDataLiteral<i64>(3LL)); + const auto data3l = pb.NewEmptyOptional(optType); + const auto data4l = pb.NewOptional(pb.NewDataLiteral<i64>(std::numeric_limits<i64>::min())); + const auto data5l = pb.NewOptional(pb.NewDataLiteral<i64>(std::numeric_limits<i64>::max())); + + const auto data1r = pb.NewDecimalLiteral(-7000000000000000000LL, 20, 18); + const auto data2r = pb.NewDecimalLiteral(3000000000000000000LL, 20, 18); + const auto data3r = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 20, 18); + const auto data4r = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 20, 18); + const auto data5r = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 20, 18); + + auto pairType = pb.NewTupleType({optType, pb.NewDecimalType(20, 18)}); + const auto list = pb.NewList(pairType, { + pb.NewTuple({data1l, data1r}), + pb.NewTuple({data1l, data2r}), + pb.NewTuple({data1l, data3r}), + pb.NewTuple({data1l, data4r}), + pb.NewTuple({data1l, data5r}), + + pb.NewTuple({data2l, data1r}), + pb.NewTuple({data2l, data2r}), + pb.NewTuple({data2l, data3r}), + pb.NewTuple({data2l, data4r}), + pb.NewTuple({data2l, data5r}), + + pb.NewTuple({data3l, data1r}), + pb.NewTuple({data3l, data2r}), + pb.NewTuple({data3l, data3r}), + pb.NewTuple({data3l, data4r}), + pb.NewTuple({data3l, data5r}), + + pb.NewTuple({data4l, data1r}), + pb.NewTuple({data4l, data2r}), + pb.NewTuple({data4l, data3r}), + pb.NewTuple({data4l, data4r}), + pb.NewTuple({data4l, data5r}), + + pb.NewTuple({data5l, data1r}), + pb.NewTuple({data5l, data2r}), + pb.NewTuple({data5l, data3r}), + pb.NewTuple({data5l, data4r}), + pb.NewTuple({data5l, data5r}), + }); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.NewTuple({ + pb.Equals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.NotEquals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.Less(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.LessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.Greater(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.GreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)) + }); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); // == + UNIT_ASSERT(!item.GetElement(1)); // != + UNIT_ASSERT(!item.GetElement(2)); // < + UNIT_ASSERT(!item.GetElement(3)); // <= + UNIT_ASSERT(!item.GetElement(4)); // > + UNIT_ASSERT(!item.GetElement(5)); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); // == + UNIT_ASSERT(!item.GetElement(1)); // != + UNIT_ASSERT(!item.GetElement(2)); // < + UNIT_ASSERT(!item.GetElement(3)); // <= + UNIT_ASSERT(!item.GetElement(4)); // > + UNIT_ASSERT(!item.GetElement(5)); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); // == + UNIT_ASSERT(!item.GetElement(1)); // != + UNIT_ASSERT(!item.GetElement(2)); // < + UNIT_ASSERT(!item.GetElement(3)); // <= + UNIT_ASSERT(!item.GetElement(4)); // > + UNIT_ASSERT(!item.GetElement(5)); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); // == + UNIT_ASSERT(!item.GetElement(1)); // != + UNIT_ASSERT(!item.GetElement(2)); // < + UNIT_ASSERT(!item.GetElement(3)); // <= + UNIT_ASSERT(!item.GetElement(4)); // > + UNIT_ASSERT(!item.GetElement(5)); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); // == + UNIT_ASSERT(!item.GetElement(1)); // != + UNIT_ASSERT(!item.GetElement(2)); // < + UNIT_ASSERT(!item.GetElement(3)); // <= + UNIT_ASSERT(!item.GetElement(4)); // > + UNIT_ASSERT(!item.GetElement(5)); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestAggrCompares) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDecimalLiteral(-7, 10, 0); + const auto data2 = pb.NewDecimalLiteral(3, 10, 0); + const auto data3 = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 10, 0); + const auto data4 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 10, 0); + const auto data5 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 10, 0); + + auto pairType = pb.NewTupleType({pb.NewDecimalType(10, 0), pb.NewDecimalType(10, 0)}); + const auto list = pb.NewList(pairType, { + pb.NewTuple({data1, data1}), + pb.NewTuple({data1, data2}), + pb.NewTuple({data1, data3}), + pb.NewTuple({data1, data4}), + pb.NewTuple({data1, data5}), + + pb.NewTuple({data2, data1}), + pb.NewTuple({data2, data2}), + pb.NewTuple({data2, data3}), + pb.NewTuple({data2, data4}), + pb.NewTuple({data2, data5}), + + pb.NewTuple({data3, data1}), + pb.NewTuple({data3, data2}), + pb.NewTuple({data3, data3}), + pb.NewTuple({data3, data4}), + pb.NewTuple({data3, data5}), + + pb.NewTuple({data4, data1}), + pb.NewTuple({data4, data2}), + pb.NewTuple({data4, data3}), + pb.NewTuple({data4, data4}), + pb.NewTuple({data4, data5}), + + pb.NewTuple({data5, data1}), + pb.NewTuple({data5, data2}), + pb.NewTuple({data5, data3}), + pb.NewTuple({data5, data4}), + pb.NewTuple({data5, data5}), + }); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.NewTuple({ + pb.AggrEquals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.AggrNotEquals(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.AggrLess(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.AggrLessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.AggrGreater(pb.Nth(item, 0), pb.Nth(item, 1)), + pb.AggrGreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)) + }); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // == + UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // != + UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // < + UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <= + UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // > + UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >= + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestIncDec) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 4, 1); + const auto data2 = pb.NewDecimalLiteral(-9999, 4, 1); + const auto data3 = pb.NewDecimalLiteral(-7, 4, 1); + const auto data4 = pb.NewDecimalLiteral(0, 4, 1); + const auto data5 = pb.NewDecimalLiteral(13, 4, 1); + const auto data6 = pb.NewDecimalLiteral(9999, 4, 1); + const auto data7 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 4, 1); + const auto data8 = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 4, 1); + + const auto list = pb.NewList(pb.NewDecimalType(4, 1), {data1, data2, data3, data4, data5, data6, data7, data8}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.NewTuple({pb.Increment(item), pb.Decrement(item)}); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -9998); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -6); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -8); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == +1); + UNIT_ASSERT(item.GetElement(1).GetInt128() == -1); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == 14); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 12); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == +NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 9998); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == +NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestMinusAbs) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDecimalLiteral(-NYql::NDecimal::Nan(), 10, 1); + const auto data1 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 10, 1); + const auto data3 = pb.NewDecimalLiteral(-7, 10, 1); + const auto data4 = pb.NewDecimalLiteral(0, 10, 1); + const auto data5 = pb.NewDecimalLiteral(13, 10, 1); + const auto data7 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 10, 1); + const auto data8 = pb.NewDecimalLiteral(+NYql::NDecimal::Nan(), 10, 1); + + const auto list = pb.NewList(pb.NewDecimalType(10, 1), {data0, data1, data3, data4, data5, data7, data8}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.NewTuple({pb.Minus(item), pb.Abs(item)}); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Nan()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == 7); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 7); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == 0); + UNIT_ASSERT(item.GetElement(1).GetInt128() == 0); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -13); + UNIT_ASSERT(item.GetElement(1).GetInt128() == +13); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf()); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan()); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFromString) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("0.0"); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("NAN"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("1.0"); + const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-.1"); + const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("3.1415926"); + const auto data5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("+inf"); + const auto data6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-INF"); + const auto data7 = pb.NewDataLiteral<NUdf::EDataSlot::String>(".123E+2"); + const auto data8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("56.78e-3"); + const auto type = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7, data8}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.StrictFromString(item, pb.NewDecimalType(10, 7)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == NYql::NDecimal::Nan()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 10000000); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -1000000); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 31415926); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == +NYql::NDecimal::Inf()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == -NYql::NDecimal::Inf()); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 123000000); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetInt128() == 567800); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestToString) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDecimalLiteral(0, 10, 7); + const auto data1 = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 10, 7); + const auto data2 = pb.NewDecimalLiteral(10000000, 10, 7); + const auto data3 = pb.NewDecimalLiteral(-1000000, 10, 7); + const auto data4 = pb.NewDecimalLiteral(31415926, 10, 7); + const auto data5 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 10, 7); + const auto data6 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 10, 7); + const auto type = pb.NewDecimalType(10, 7); + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.ToString(item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "0"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "nan"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "1"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "-0.1"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "3.1415926"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "inf"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "-inf"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFromStringToDouble) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto type = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("0"); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("+3.332873"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-3.332873"); + const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("+3.1415926"); + const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-3.1415926"); + + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Convert(pb.FromString(item, pb.NewDecimalType(35,25)), pb.NewDataType(NUdf::TDataType<double>::Id)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<double>(), 0.); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<double>(), +3.332873); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<double>(), -3.332873); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<double>(), +3.1415926); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<double>(), -3.1415926); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFromUtf8ToFloat) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto type = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id); + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("0"); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("+24.75"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("-24.75"); + const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("+42.42"); + const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("-42.42"); + + const auto list = pb.NewList(type, {data0, data1, data2, data3, data4}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.Convert(pb.FromString(item, pb.NewDecimalType(35,25)), pb.NewDataType(NUdf::TDataType<float>::Id)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<float>(), 0.f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<float>(), +24.75f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<float>(), -24.75f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<float>(), +42.42f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<float>(), -42.42f); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp new file mode 100644 index 00000000000..ac6d4c356b2 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp @@ -0,0 +1,393 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> + +namespace NKikimr { +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLDictRelatedNodesTest) { + Y_UNIT_TEST_LLVM(TestDictLength) { + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + const auto key1 = pgmBuilder.NewDataLiteral<ui32>(1); + const auto key2 = pgmBuilder.NewDataLiteral<ui32>(2); + const auto key3 = pgmBuilder.NewDataLiteral<ui32>(2); + const auto payload1 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("C"); + TVector<std::pair<TRuntimeNode, TRuntimeNode>> dictItems; + dictItems.push_back(std::make_pair(key1, payload1)); + dictItems.push_back(std::make_pair(key2, payload2)); + dictItems.push_back(std::make_pair(key3, payload3)); + const auto dictType = pgmBuilder.NewDictType(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id), + pgmBuilder.NewDataType(NUdf::TDataType<char*>::Id), false); + const auto dict = pgmBuilder.NewDict(dictType, dictItems); + const auto pgmReturn = pgmBuilder.Length(dict); + + const auto graph = setup.BuildGraph(pgmReturn); + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().template Get<ui64>(), 2); + } + + Y_UNIT_TEST_LLVM(TestDictContains) { + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + const auto key1 = pgmBuilder.NewDataLiteral<ui32>(1); + const auto key2 = pgmBuilder.NewDataLiteral<ui32>(2); + const auto key3 = pgmBuilder.NewDataLiteral<ui32>(2); + const auto missingKey = pgmBuilder.NewDataLiteral<ui32>(3); + const auto payload1 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("C"); + TVector<std::pair<TRuntimeNode, TRuntimeNode>> dictItems; + dictItems.push_back(std::make_pair(key1, payload1)); + dictItems.push_back(std::make_pair(key2, payload2)); + dictItems.push_back(std::make_pair(key3, payload3)); + const auto dictType = pgmBuilder.NewDictType(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id), + pgmBuilder.NewDataType(NUdf::TDataType<char*>::Id), false); + const auto dict = pgmBuilder.NewDict(dictType, dictItems); + const auto keys = pgmBuilder.NewList(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id), {key1, key2, missingKey}); + + const auto pgmReturn = pgmBuilder.Map(keys, + [&](TRuntimeNode key) { + return pgmBuilder.Contains(dict, key); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestDictLookup) { + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + const auto key1 = pgmBuilder.NewDataLiteral<ui32>(1); + const auto key2 = pgmBuilder.NewDataLiteral<ui32>(2); + const auto key3 = pgmBuilder.NewDataLiteral<ui32>(2); + const auto missingKey = pgmBuilder.NewDataLiteral<ui32>(3); + const auto payload1 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("C"); + TVector<std::pair<TRuntimeNode, TRuntimeNode>> dictItems; + dictItems.push_back(std::make_pair(key1, payload1)); + dictItems.push_back(std::make_pair(key2, payload2)); + dictItems.push_back(std::make_pair(key3, payload3)); + const auto dictType = pgmBuilder.NewDictType(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id), + pgmBuilder.NewDataType(NUdf::TDataType<char*>::Id), false); + const auto dict = pgmBuilder.NewDict(dictType, dictItems); + const auto keys = pgmBuilder.NewList(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id), {key1, key2, missingKey}); + + const auto pgmReturn = pgmBuilder.Map(keys, + [&](TRuntimeNode key) { + return pgmBuilder.Lookup(dict, key); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item); + UNBOXED_VALUE_STR_EQUAL(item, "A"); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item); + UNBOXED_VALUE_STR_EQUAL(item, "B"); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + template<bool Multi> + TRuntimeNode PrepareTestDict(TProgramBuilder& pgmBuilder, TRuntimeNode(TProgramBuilder::* factory)(TRuntimeNode list, bool multi, + const TProgramBuilder::TUnaryLambda& keySelector, + const TProgramBuilder::TUnaryLambda& payloadSelector, bool isCompact, ui64 itemsCountHint)) { + const auto key1 = pgmBuilder.NewDataLiteral<ui32>(1); + const auto key2 = pgmBuilder.NewDataLiteral<ui32>(2); + const auto key3 = pgmBuilder.NewDataLiteral<ui32>(2); + const auto key4 = pgmBuilder.NewDataLiteral<ui32>(5); + const auto key5 = pgmBuilder.NewDataLiteral<ui32>(7); + const auto payload1 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("D"); + const auto payload5 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("E"); + auto structType = pgmBuilder.NewStructType(pgmBuilder.NewEmptyStructType(), "Key", pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); + structType = pgmBuilder.NewStructType(structType, "Payload", pgmBuilder.NewDataType(NUdf::TDataType<char*>::Id)); + const auto list = pgmBuilder.NewList(structType, { + pgmBuilder.AddMember(pgmBuilder.AddMember(pgmBuilder.NewEmptyStruct(), "Key", key3), "Payload", payload3), + pgmBuilder.AddMember(pgmBuilder.AddMember(pgmBuilder.NewEmptyStruct(), "Key", key1), "Payload", payload1), + pgmBuilder.AddMember(pgmBuilder.AddMember(pgmBuilder.NewEmptyStruct(), "Key", key5), "Payload", payload5), + pgmBuilder.AddMember(pgmBuilder.AddMember(pgmBuilder.NewEmptyStruct(), "Key", key4), "Payload", payload4), + pgmBuilder.AddMember(pgmBuilder.AddMember(pgmBuilder.NewEmptyStruct(), "Key", key2), "Payload", payload2) + }); + const auto dict = (pgmBuilder.*factory)(list, Multi, + [&](TRuntimeNode item) { + return pgmBuilder.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pgmBuilder.Member(item, "Payload"); + }, false, 0); + return dict; + } + + template<bool LLVM> + void TestConvertedDictContains(TRuntimeNode(TProgramBuilder::* factory)(TRuntimeNode list, bool multi, + const TProgramBuilder::TUnaryLambda& keySelector, + const TProgramBuilder::TUnaryLambda& payloadSelector, bool isCompact, ui64 itemsCountHint)) { + + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + const auto dict = PrepareTestDict<false>(pgmBuilder, factory); + + const auto key1 = pgmBuilder.NewDataLiteral<ui32>(1); + const auto key2 = pgmBuilder.NewDataLiteral<ui32>(2); + const auto missingKey = pgmBuilder.NewDataLiteral<ui32>(42); + const auto keys = pgmBuilder.NewList(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id), {key1, key2, missingKey}); + + const auto pgmReturn = pgmBuilder.Map(keys, + [&](TRuntimeNode key) { + return pgmBuilder.Contains(dict, key); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + template<bool LLVM> + void TestConvertedDictLookup(TRuntimeNode(TProgramBuilder::* factory)(TRuntimeNode list, bool multi, + const TProgramBuilder::TUnaryLambda& keySelector, + const TProgramBuilder::TUnaryLambda& payloadSelector, bool isCompact, ui64 itemsCountHint)) { + + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + const auto dict = PrepareTestDict<false>(pgmBuilder, factory); + + const auto key1 = pgmBuilder.NewDataLiteral<ui32>(1); + const auto key2 = pgmBuilder.NewDataLiteral<ui32>(2); + const auto missingKey = pgmBuilder.NewDataLiteral<ui32>(18); + const auto keys = pgmBuilder.NewList(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id), {key1, key2, missingKey}); + + const auto pgmReturn = pgmBuilder.Map(keys, + [&](TRuntimeNode key) { + return pgmBuilder.Lookup(dict, key); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item); + UNBOXED_VALUE_STR_EQUAL(item, "A"); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item); + UNBOXED_VALUE_STR_EQUAL(item, "C"); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestSortedDictContains) { + TestConvertedDictContains<LLVM>(&TProgramBuilder::ToSortedDict); + } + + Y_UNIT_TEST_LLVM(TestSortedDictLookup) { + TestConvertedDictLookup<LLVM>(&TProgramBuilder::ToSortedDict); + } + + Y_UNIT_TEST_LLVM(TestHashedDictContains) { + TestConvertedDictContains<LLVM>(&TProgramBuilder::ToHashedDict); + } + + Y_UNIT_TEST_LLVM(TestHashedDictLookup) { + TestConvertedDictLookup<LLVM>(&TProgramBuilder::ToHashedDict); + } + + template<bool LLVM, bool SortBeforeCompare> + void TestDictItemsImpl(TRuntimeNode(TProgramBuilder::* factory)(TRuntimeNode list, bool multi, + const TProgramBuilder::TUnaryLambda& keySelector, + const TProgramBuilder::TUnaryLambda& payloadSelector, bool isCompact, ui64 itemsCountHint)) { + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + const auto dict = PrepareTestDict<false>(pgmBuilder, factory); + const auto pgmReturn = pgmBuilder.DictItems(dict); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + + std::vector<std::pair<ui32, TString>> items; + for (NUdf::TUnboxedValue item; iterator.Next(item);) { + const auto& pay = item.GetElement(1); + items.emplace_back(item.GetElement(0).template Get<ui32>(), pay.AsStringRef()); + } + + if (SortBeforeCompare) { + std::sort(items.begin(), items.end(), [](const std::pair<ui32, TString>& left, const std::pair<ui32, TString>& right) { + return left.first < right.first; + }); + } + + UNIT_ASSERT_VALUES_EQUAL(items.size(), 4U); + UNIT_ASSERT_VALUES_EQUAL(items[0].first, 1); + UNIT_ASSERT_VALUES_EQUAL(items[0].second, "A"); + UNIT_ASSERT_VALUES_EQUAL(items[1].first, 2); + UNIT_ASSERT_VALUES_EQUAL(items[1].second, "C"); + UNIT_ASSERT_VALUES_EQUAL(items[2].first, 5); + UNIT_ASSERT_VALUES_EQUAL(items[2].second, "D"); + UNIT_ASSERT_VALUES_EQUAL(items[3].first, 7); + UNIT_ASSERT_VALUES_EQUAL(items[3].second, "E"); + } + + template<bool LLVM, bool SortBeforeCompare> + void TestDictKeysImpl(TRuntimeNode(TProgramBuilder::* factory)(TRuntimeNode list, bool multi, + const TProgramBuilder::TUnaryLambda& keySelector, + const TProgramBuilder::TUnaryLambda& payloadSelector, bool isCompact, ui64 itemsCountHint)) { + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + const auto dict = PrepareTestDict<false>(pgmBuilder, factory); + const auto pgmReturn = pgmBuilder.DictKeys(dict); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + + std::vector<ui32> items; + for (NUdf::TUnboxedValue item; iterator.Next(item);) { + items.emplace_back(item.template Get<ui32>()); + } + + if (SortBeforeCompare) { + std::sort(items.begin(), items.end()); + } + + UNIT_ASSERT_VALUES_EQUAL(items.size(), 4U); + UNIT_ASSERT_VALUES_EQUAL(items[0], 1); + UNIT_ASSERT_VALUES_EQUAL(items[1], 2); + UNIT_ASSERT_VALUES_EQUAL(items[2], 5); + UNIT_ASSERT_VALUES_EQUAL(items[3], 7); + } + + template<bool LLVM, bool SortBeforeCompare> + void TestDictPayloadsImpl(TRuntimeNode(TProgramBuilder::* factory)(TRuntimeNode list, bool multi, + const TProgramBuilder::TUnaryLambda& keySelector, + const TProgramBuilder::TUnaryLambda& payloadSelector, bool isCompact, ui64 itemsCountHint)) { + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + const auto dict = PrepareTestDict<false>(pgmBuilder, factory); + const auto pgmReturn = pgmBuilder.DictPayloads(dict); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + + std::vector<TString> items; + for (NUdf::TUnboxedValue item; iterator.Next(item);) { + items.emplace_back(item.AsStringRef()); + } + + if (SortBeforeCompare) { + std::sort(items.begin(), items.end()); + } + + UNIT_ASSERT_VALUES_EQUAL(items.size(), 4U); + UNIT_ASSERT_VALUES_EQUAL(items[0], "A"); + UNIT_ASSERT_VALUES_EQUAL(items[1], "C"); + UNIT_ASSERT_VALUES_EQUAL(items[2], "D"); + UNIT_ASSERT_VALUES_EQUAL(items[3], "E"); + } + + Y_UNIT_TEST_LLVM(TestSortedDictItems) { + TestDictItemsImpl<LLVM, false>(&TProgramBuilder::ToSortedDict); + } + + Y_UNIT_TEST_LLVM(TestHashedDictItems) { + TestDictItemsImpl<LLVM, true>(&TProgramBuilder::ToHashedDict); + } + + Y_UNIT_TEST_LLVM(TestSortedDictKeys) { + TestDictKeysImpl<LLVM, false>(&TProgramBuilder::ToSortedDict); + } + + Y_UNIT_TEST_LLVM(TestHashedDictKeys) { + TestDictKeysImpl<LLVM, true>(&TProgramBuilder::ToHashedDict); + } + + Y_UNIT_TEST_LLVM(TestSortedPayloadsKeys) { + TestDictPayloadsImpl<LLVM, false>(&TProgramBuilder::ToSortedDict); + } + + Y_UNIT_TEST_LLVM(TestHashedPayloadsKeys) { + TestDictPayloadsImpl<LLVM, true>(&TProgramBuilder::ToHashedDict); + } + + template<bool LLVM> + void TestConvertedMultiDictLookup(TRuntimeNode(TProgramBuilder::* factory)(TRuntimeNode list, bool multi, + const TProgramBuilder::TUnaryLambda& keySelector, + const TProgramBuilder::TUnaryLambda& payloadSelector, bool isCompact, ui64 itemsCountHint)) { + + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + const auto dict = PrepareTestDict<true>(pgmBuilder, factory); + const auto key1 = pgmBuilder.NewDataLiteral<ui32>(1); + const auto key2 = pgmBuilder.NewDataLiteral<ui32>(2); + const auto missingKey = pgmBuilder.NewDataLiteral<ui32>(3); + const auto keys = pgmBuilder.NewList(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id), {key1, key2, missingKey}); + const auto pgmReturn = pgmBuilder.Map(keys, + [&](TRuntimeNode key) { + return pgmBuilder.Lookup(dict, key); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item, item2; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item); + auto iter2 = item.GetListIterator(); + UNIT_ASSERT(iter2.Next(item2)); + UNBOXED_VALUE_STR_EQUAL(item2, "A"); + UNIT_ASSERT(!iter2.Next(item2)); + UNIT_ASSERT(!iter2.Next(item2)); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item); + iter2 = item.GetListIterator(); + UNIT_ASSERT(iter2.Next(item2)); + UNBOXED_VALUE_STR_EQUAL(item2, "C"); + UNIT_ASSERT(iter2.Next(item2)); + UNBOXED_VALUE_STR_EQUAL(item2, "B"); + UNIT_ASSERT(!iter2.Next(item2)); + UNIT_ASSERT(!iter2.Next(item2)); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestSortedMultiDictLookup) { + TestConvertedMultiDictLookup<LLVM>(&TProgramBuilder::ToSortedDict); + } + + Y_UNIT_TEST_LLVM(TestHashedMultiDictLookup) { + TestConvertedMultiDictLookup<LLVM>(&TProgramBuilder::ToHashedDict); + } +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_filters_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_filters_ut.cpp new file mode 100644 index 00000000000..e2c1c4dc690 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_filters_ut.cpp @@ -0,0 +1,1120 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/mkql_runtime_version.h> +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> + +namespace NKikimr { +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLFiltersTest) { + Y_UNIT_TEST_LLVM(TestSkipNullMembers) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto structType = pb.NewStructType({{"Key", dataType}, {"Payload", dataType}}); + + const auto data1 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewDataLiteral<i32>(1))}, {"Payload", pb.NewEmptyOptional(dataType)}}); + const auto data2 = pb.NewStruct(structType, {{"Key", pb.NewEmptyOptional(dataType)}, {"Payload", pb.NewOptional(pb.NewDataLiteral<i32>(2))}}); + const auto data3 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewDataLiteral<i32>(3))}, {"Payload", pb.NewEmptyOptional(dataType)}}); + + const auto list = pb.NewList(structType, {data1, data2, data3}); + + const auto pgmReturn = pb.SkipNullMembers(list, {"Payload"}); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterNullMembers) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto structType = pb.NewStructType({{"Key", dataType}, {"Payload", dataType}}); + + const auto data1 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewDataLiteral<i32>(1))}, {"Payload", pb.NewEmptyOptional(dataType)}}); + const auto data2 = pb.NewStruct(structType, {{"Key", pb.NewEmptyOptional(dataType)}, {"Payload", pb.NewOptional(pb.NewDataLiteral<i32>(2))}}); + const auto data3 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewDataLiteral<i32>(3))}, {"Payload", pb.NewEmptyOptional(dataType)}}); + + const auto list = pb.NewList(structType, {data1, data2, data3}); + + const auto pgmReturn = pb.FilterNullMembers(list, {"Payload"}); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterNullMembersMultiOptional) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id))); + const auto structType = pb.NewStructType({{"Key", dataType}, {"Payload", dataType}}); + const auto justNothing = pb.NewOptional(pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i32>::Id)); + + const auto data1 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(1)))}, {"Payload", pb.NewEmptyOptional(dataType)}}); + const auto data2 = pb.NewStruct(structType, {{"Key", pb.NewEmptyOptional(dataType)}, {"Payload", pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(2)))}}); + const auto data3 = pb.NewStruct(structType, {{"Key", justNothing}, {"Payload", justNothing}}); + + const auto list = pb.NewList(structType, {data1, data2, data3}); + + const auto pgmReturn = pb.FilterNullMembers(list, {"Payload"}); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestSkipNullMembersOverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto structType = pb.NewStructType({{"Key", dataType}, {"Payload", dataType}}); + + const auto data1 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewDataLiteral<i32>(1))}, {"Payload", pb.NewEmptyOptional(dataType)}}); + const auto data2 = pb.NewStruct(structType, {{"Key", pb.NewEmptyOptional(dataType)}, {"Payload", pb.NewOptional(pb.NewDataLiteral<i32>(2))}}); + const auto data3 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewDataLiteral<i32>(3))}, {"Payload", pb.NewEmptyOptional(dataType)}}); + + const auto list = pb.NewList(structType, {data1, data2, data3}); + + const auto pgmReturn = pb.FromFlow(pb.SkipNullMembers(pb.ToFlow(list), {"Payload"})); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterNullMembersOverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto structType = pb.NewStructType({{"Key", dataType}, {"Payload", dataType}}); + + const auto data1 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewDataLiteral<i32>(1))}, {"Payload", pb.NewEmptyOptional(dataType)}}); + const auto data2 = pb.NewStruct(structType, {{"Key", pb.NewEmptyOptional(dataType)}, {"Payload", pb.NewOptional(pb.NewDataLiteral<i32>(2))}}); + const auto data3 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewDataLiteral<i32>(3))}, {"Payload", pb.NewEmptyOptional(dataType)}}); + + const auto list = pb.NewList(structType, {data1, data2, data3}); + + const auto pgmReturn = pb.FromFlow(pb.FilterNullMembers(pb.ToFlow(list), {"Payload"})); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterNullMembersMultiOptionalOverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id))); + const auto structType = pb.NewStructType({{"Key", dataType}, {"Payload", dataType}}); + const auto justNothing = pb.NewOptional(pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i32>::Id)); + + const auto data1 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(1)))}, {"Payload", pb.NewEmptyOptional(dataType)}}); + const auto data2 = pb.NewStruct(structType, {{"Key", pb.NewEmptyOptional(dataType)}, {"Payload", pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(2)))}}); + const auto data3 = pb.NewStruct(structType, {{"Key", justNothing}, {"Payload", justNothing}}); + + const auto list = pb.NewList(structType, {data1, data2, data3}); + + const auto pgmReturn = pb.FromFlow(pb.FilterNullMembers(pb.ToFlow(list), {"Payload"})); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestSkipNullElements) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.SkipNullElements(list, {1U, 2U}); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -2); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterNullElements) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.FilterNullElements(list, {1U, 2U}); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -2); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterNullElementsMultiOptional) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id))); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + const auto justNothing = pb.NewOptional(pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i32>::Id)); + + const auto data1 = pb.NewTuple(tupleType, {justNothing, pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(-1)))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(2))), justNothing}); + const auto data3 = pb.NewTuple(tupleType, {justNothing, pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(-3)))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(4))), justNothing, pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(-4)))}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.FilterNullElements(list, {1U, 2U}); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2); + UNIT_ASSERT(!item.GetElement(2)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 4); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -4); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestSkipNullElementsOverOverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.FromFlow(pb.SkipNullElements(pb.ToFlow(list), {1U, 2U})); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -2); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterNullElementsOverOverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.FromFlow(pb.FilterNullElements(pb.ToFlow(list), {1U, 2U})); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -2); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterNullElementsOverMultiOptionalOverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id))); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + const auto justNothing = pb.NewOptional(pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i32>::Id)); + + const auto data1 = pb.NewTuple(tupleType, {justNothing, pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(-1)))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(2))), justNothing}); + const auto data3 = pb.NewTuple(tupleType, {justNothing, pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(-3)))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(4))), justNothing, pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(-4)))}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.FromFlow(pb.FilterNullElements(pb.ToFlow(list), {1U, 2U})); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2); + UNIT_ASSERT(!item.GetElement(2)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 4); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -4); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterOverList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto optionalType = pb.NewOptionalType(dataType); + const auto list = pb.NewList(optionalType, {pb.NewEmptyOptional(optionalType), pb.NewOptional(data2)}); + + const auto pgmReturn = pb.Filter(list, + [&](TRuntimeNode item) { + return pb.Exists(item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterOverStream) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<ui32>(0); + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto data3 = pb.NewDataLiteral<ui32>(3); + const auto data4 = pb.NewDataLiteral<ui32>(4); + const auto data5 = pb.NewDataLiteral<ui32>(5); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5}); + const auto pgmReturn = pb.Filter(pb.Iterator(list, {}), + [&](TRuntimeNode item) { + return pb.Greater(pb.Unwrap(pb.Mod(item, data3), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0), data1); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 5); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterOverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<ui32>(0); + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto data3 = pb.NewDataLiteral<ui32>(3); + const auto data4 = pb.NewDataLiteral<ui32>(4); + const auto data5 = pb.NewDataLiteral<ui32>(5); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5}); + const auto pgmReturn = pb.FromFlow(pb.Filter(pb.ToFlow(list), + [&](TRuntimeNode item) { + return pb.Greater(pb.Unwrap(pb.Mod(item, data3), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0), data1); + })); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 5); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterOverListLazy) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<ui32>::Id)); + const auto data = pb.NewEmptyOptional(dataType); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2U)); + const auto list = pb.NewList(dataType, {data, data2}); + + const auto pgmReturn = pb.Filter(pb.LazyList(list), + [&](TRuntimeNode item) { return pb.Exists(item); } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(item); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterByString) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto optionalType = pb.NewOptionalType(dataType); + const auto data0 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("000")); + const auto data1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("100")); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("200")); + const auto data3 = pb.NewEmptyOptional(optionalType); + const auto list = pb.NewList(optionalType, {data0, data1, data2, data3, data1, data0}); + + const auto pgmReturn = pb.Filter(list, + [&](TRuntimeNode item) { + return pb.Coalesce(pb.Equals(item, data1), pb.NewDataLiteral<bool>(false)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto result = graph->GetValue(); + UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), 2ULL); + UNBOXED_VALUE_STR_EQUAL(result.GetElement(0U), "100"); + UNBOXED_VALUE_STR_EQUAL(result.GetElement(1U), "100"); + } + + Y_UNIT_TEST_LLVM(TestSkipWhile) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000"); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200"); + const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300"); + const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3}); + + const auto pgmReturn = pb.SkipWhile(list, + [&](TRuntimeNode item) { + return pb.NotEquals(item, data2); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "200"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "300"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestTakeWhile) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000"); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200"); + const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300"); + const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3}); + + const auto pgmReturn = pb.TakeWhile(list, + [&](TRuntimeNode item) { + return pb.NotEquals(item, data2); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "000"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "100"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestSkipWhileOverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000"); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200"); + const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300"); + const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3}); + + const auto pgmReturn = pb.FromFlow(pb.SkipWhile(pb.ToFlow(list), + [&](TRuntimeNode item) { + return pb.NotEquals(item, data2); + })); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNBOXED_VALUE_STR_EQUAL(item, "200"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNBOXED_VALUE_STR_EQUAL(item, "300"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestTakeWhileOverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000"); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200"); + const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300"); + const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3}); + + const auto pgmReturn = pb.FromFlow(pb.TakeWhile(pb.ToFlow(list), + [&](TRuntimeNode item) { + return pb.NotEquals(item, data2); + })); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNBOXED_VALUE_STR_EQUAL(item, "000"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNBOXED_VALUE_STR_EQUAL(item, "100"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestSkipWhileInclusive) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000"); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200"); + const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300"); + const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3}); + + const auto pgmReturn = pb.SkipWhileInclusive(list, + [&](TRuntimeNode item) { + return pb.NotEquals(item, data2); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "300"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestTakeWhileInclusive) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000"); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200"); + const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300"); + const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3}); + + const auto pgmReturn = pb.TakeWhileInclusive(list, + [&](TRuntimeNode item) { + return pb.NotEquals(item, data2); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "000"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "100"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "200"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestSkipWhileInclusiveOnEmptyList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto list = pb.NewList(dataType, {}); + + const auto pgmReturn = pb.SkipWhileInclusive(list, + [&](TRuntimeNode item) { + return pb.NotEquals(item, pb.NewDataLiteral<NUdf::EDataSlot::String>("000")); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestTakeWhileInclusiveOnEmptyList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto list = pb.NewList(dataType, {}); + + const auto pgmReturn = pb.TakeWhileInclusive(list, + [&](TRuntimeNode item) { + return pb.NotEquals(item, pb.NewDataLiteral<NUdf::EDataSlot::String>("000")); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestSkipWhileInclusiveOverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000"); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200"); + const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300"); + const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3}); + + const auto pgmReturn = pb.FromFlow(pb.SkipWhileInclusive(pb.ToFlow(list), + [&](TRuntimeNode item) { + return pb.NotEquals(item, data2); + })); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNBOXED_VALUE_STR_EQUAL(item, "300"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestTakeWhileInclusiveOverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000"); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200"); + const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300"); + const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3}); + + const auto pgmReturn = pb.FromFlow(pb.TakeWhileInclusive(pb.ToFlow(list), + [&](TRuntimeNode item) { + return pb.NotEquals(item, data2); + })); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNBOXED_VALUE_STR_EQUAL(item, "000"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNBOXED_VALUE_STR_EQUAL(item, "100"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNBOXED_VALUE_STR_EQUAL(item, "200"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestDateToStringCompleteCheck) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list = pb.ListFromRange(pb.NewDataLiteral<ui16>(0U), pb.NewDataLiteral<ui16>(NUdf::MAX_DATE), pb.NewDataLiteral<ui16>(1U)); + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true); + const auto pgmReturn = pb.Not(pb.HasItems(pb.Filter(list, + [&](TRuntimeNode item) { + const auto date = pb.ToIntegral(item, dateType); + const auto utf8 = pb.ToString<true>(date); + return pb.AggrNotEquals(date, pb.FromString(utf8, dateType)); + } + ))); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto result = graph->GetValue(); + UNIT_ASSERT(result.template Get<bool>()); + } + + Y_UNIT_TEST_LLVM(TestTzDateToStringCompleteCheck) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list = pb.ListFromRange(pb.NewDataLiteral<ui16>(0U), pb.NewDataLiteral<ui16>(NUdf::MAX_DATE), pb.NewDataLiteral<ui16>(1U)); + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true); + const auto dateTypeTz = pb.NewDataType(NUdf::EDataSlot::TzDate, true); + const auto canada = pb.NewDataLiteral<ui16>(375U); + const auto europe = pb.NewDataLiteral<ui16>(459U); + const auto pgmReturn = pb.Not(pb.HasItems(pb.Filter(list, + [&](TRuntimeNode item) { + const auto date = pb.Unwrap(pb.ToIntegral(item, dateType), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0); + const auto date1 = pb.Unwrap(pb.FromString(pb.ToString<true>(pb.AddTimezone(date, canada)), dateTypeTz), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0); + const auto date2 = pb.Unwrap(pb.FromString(pb.ToString<true>(pb.AddTimezone(date, europe)), dateTypeTz), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0); + return pb.Or({pb.NotEquals(date, date1), pb.NotEquals(date, date2)}); + } + ))); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto result = graph->GetValue(); + UNIT_ASSERT(result.template Get<bool>()); + } + + Y_UNIT_TEST_LLVM(TestInt16ToFloatCompleteCheck) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list = pb.ListFromRange(pb.NewDataLiteral<i16>(std::numeric_limits<i16>::min()), pb.NewDataLiteral<i16>(std::numeric_limits<i16>::max()), pb.NewDataLiteral<i16>(1)); + const auto type = pb.NewDataType(NUdf::EDataSlot::Float); + const auto pgmReturn = pb.Not(pb.HasItems(pb.Filter(list, + [&](TRuntimeNode item) { + return pb.NotEquals(item, pb.Convert(item, type)); + } + ))); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto result = graph->GetValue(); + UNIT_ASSERT(result.template Get<bool>()); + } + + Y_UNIT_TEST_LLVM(TestUint16ToFloatCompleteCheck) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list = pb.ListFromRange(pb.NewDataLiteral<ui16>(std::numeric_limits<ui16>::min()), pb.NewDataLiteral<ui16>(std::numeric_limits<ui16>::max()), pb.NewDataLiteral<ui16>(1U)); + const auto type = pb.NewDataType(NUdf::EDataSlot::Float); + const auto pgmReturn = pb.Not(pb.HasItems(pb.Filter(list, + [&](TRuntimeNode item) { + return pb.NotEquals(item, pb.Convert(item, type)); + } + ))); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto result = graph->GetValue(); + UNIT_ASSERT(result.template Get<bool>()); + } + + Y_UNIT_TEST_LLVM(TestDateToDatetimeCompleteCheck) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list = pb.ListFromRange(pb.NewDataLiteral<ui16>(0U), pb.NewDataLiteral<ui16>(NUdf::MAX_DATE), pb.NewDataLiteral<ui16>(1U)); + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true); + const auto datetimeType = pb.NewDataType(NUdf::EDataSlot::Datetime, true); + const auto pgmReturn = pb.Not(pb.HasItems(pb.Filter(list, + [&](TRuntimeNode item) { + const auto date = pb.ToIntegral(item, dateType); + return pb.Coalesce(pb.NotEquals(date, pb.Convert(date, datetimeType)), pb.NewDataLiteral(false)); + } + ))); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto result = graph->GetValue(); + UNIT_ASSERT(result.template Get<bool>()); + } + + Y_UNIT_TEST_LLVM(TestTzDateToDatetimeCompleteCheck) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list = pb.ListFromRange(pb.NewDataLiteral<ui16>(0U), pb.NewDataLiteral<ui16>(NUdf::MAX_DATE), pb.NewDataLiteral<ui16>(1U)); + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true); + const auto datetimeType = pb.NewDataType(NUdf::EDataSlot::Datetime); + const auto canada = pb.NewDataLiteral<ui16>(375U); + const auto europe = pb.NewDataLiteral<ui16>(459U); + const auto pgmReturn = pb.Not(pb.HasItems(pb.Filter(list, + [&](TRuntimeNode item) { + const auto date = pb.ToIntegral(item, dateType); + const auto date1 = pb.Unwrap(pb.AddTimezone(date, canada), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0); + const auto date2 = pb.Unwrap(pb.AddTimezone(date, europe), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0); + return pb.Or({pb.NotEquals(date1, pb.Cast(date1, datetimeType)), pb.NotEquals(date2, pb.Cast(date2, datetimeType))}); + } + ))); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto result = graph->GetValue(); + UNIT_ASSERT(result.template Get<bool>()); + } + + Y_UNIT_TEST_LLVM(TestDateAddTimezoneAndCastOrderCompleteCheck) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list = pb.ListFromRange(pb.NewDataLiteral<ui16>(0U), pb.NewDataLiteral<ui16>(NUdf::MAX_DATE), pb.NewDataLiteral<ui16>(1U)); + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true); + const auto datetimeType = pb.NewDataType(NUdf::EDataSlot::Datetime); + const auto datetimeTypeTz = pb.NewDataType(NUdf::EDataSlot::TzDatetime); + const auto canada = pb.NewDataLiteral<ui16>(375U); + const auto europe = pb.NewDataLiteral<ui16>(459U); + const auto pgmReturn = pb.Not(pb.HasItems(pb.Filter(list, + [&](TRuntimeNode item) { + const auto date = pb.Unwrap(pb.ToIntegral(item, dateType), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0); + const auto date1_1 = pb.Cast(pb.Unwrap(pb.AddTimezone(date, canada), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0), datetimeTypeTz); + const auto date1_2 = pb.Unwrap(pb.AddTimezone(pb.Cast(date, datetimeType), canada), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0); + const auto date2_1 = pb.Cast(pb.Unwrap(pb.AddTimezone(date, europe), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0), datetimeTypeTz); + const auto date2_2 = pb.Unwrap(pb.AddTimezone(pb.Cast(date, datetimeType), europe), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0); + return pb.Or({pb.NotEquals(date1_1, date1_2), pb.NotEquals(date2_1, date2_2), pb.NotEquals(date1_1, date2_2), pb.NotEquals(date2_1, date1_2)}); + } + ))); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto result = graph->GetValue(); + UNIT_ASSERT(result.template Get<bool>()); + } + + Y_UNIT_TEST_LLVM(TestFilterWithLimitOverList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<ui32>(0); + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto data3 = pb.NewDataLiteral<ui32>(3); + const auto data4 = pb.NewDataLiteral<ui32>(4); + const auto data5 = pb.NewDataLiteral<ui32>(5); + const auto data6 = pb.NewDataLiteral<ui32>(6); + const auto limit = pb.NewDataLiteral<ui64>(3ULL); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto optionalType = pb.NewOptionalType(dataType); + const auto list = pb.NewList(optionalType, { + pb.NewOptional(data0), + pb.NewOptional(data1), + pb.NewOptional(data2), + pb.NewOptional(data3), + pb.NewOptional(data4), + pb.NewOptional(data5), + pb.NewOptional(data3), + pb.NewOptional(data4), + pb.NewOptional(data6), + pb.NewEmptyOptional(optionalType), + pb.NewEmptyOptional(optionalType) + }); + + const auto pgmReturn = pb.Filter(list, limit, + [&](TRuntimeNode item) { + return pb.AggrEquals(pb.Unwrap(pb.Mod(item, data2), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0), data0); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterWithLimitOverEmptyList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto limit = pb.NewDataLiteral<ui64>(3ULL); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto optionalType = pb.NewOptionalType(dataType); + const auto list = pb.NewEmptyList(optionalType); + + const auto pgmReturn = pb.Filter(list, limit, + [&](TRuntimeNode) { + return pb.NewDataLiteral<bool>(true); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterWithLimitOverLazyList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto start = pb.NewDataLiteral<ui64>(0ULL); + const auto stop = pb.NewDataLiteral<ui64>(1000000000000ULL); + const auto step = pb.NewDataLiteral<ui64>(1ULL); + + const auto limit = pb.NewDataLiteral<ui64>(7ULL); + const auto list = pb.ListFromRange(start, stop, step); + + const auto pgmReturn = pb.Filter(list, limit, + [&](TRuntimeNode item) { + return pb.AggrEquals(pb.CountBits(item), pb.NewDataLiteral<ui64>(3ULL)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 7ULL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 11ULL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 13ULL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 14ULL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 19ULL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 21ULL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 22ULL); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterWithSmallLimitGetTailOfLargeList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto start = pb.NewDataLiteral<ui64>(0ULL); + const auto stop = pb.NewDataLiteral<ui64>(100100LL); + const auto step = pb.NewDataLiteral<ui64>(1ULL); + + const auto limit = pb.NewDataLiteral<ui64>(3ULL); + const auto list = pb.ListFromRange(start, stop, step); + + const auto pgmReturn = pb.Filter(pb.Collect(list), limit, + [&](TRuntimeNode item) { + return pb.AggrGreaterOrEqual(item, pb.NewDataLiteral<ui64>(100000LL)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 100000LL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 100001LL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 100002LL); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterWithLimitOverEnumerate) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto start = pb.NewDataLiteral<i64>(0LL); + const auto stop = pb.NewDataLiteral<i64>(-1000000000000LL); + const auto step = pb.NewDataLiteral<i64>(-1LL); + + const auto estart = pb.NewDataLiteral<ui64>(42ULL); + const auto estep = pb.NewDataLiteral<ui64>(3ULL); + + const auto limit = pb.NewDataLiteral<ui64>(5ULL); + const auto list = pb.Enumerate(pb.ListFromRange(start, stop, step), estart, estep); + + const auto pgmReturn = pb.Filter(list, limit, + [&](TRuntimeNode item) { + return pb.AggrEquals(pb.CountBits(pb.Add(pb.Nth(item, 0), pb.Nth(item, 1))), pb.NewDataLiteral<i64>(4LL)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui64>(), 48ULL); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<i64>(), -2LL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui64>(), 60ULL); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<i64>(), -6LL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui64>(), 66ULL); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<i64>(), -8LL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui64>(), 69ULL); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<i64>(), -9LL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui64>(), 96ULL); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<i64>(), -18LL); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterWithLimitOverStream) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto type = pb.NewStreamType(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<ui32>::Id))); + const auto stream = pb.ChainMap(pb.SourceOf(type), pb.NewDataLiteral<ui64>(0ULL), + [&](TRuntimeNode, TRuntimeNode state) -> TRuntimeNodePair { + return {state, pb.Increment(state)}; + } + ); + + const auto limit = pb.NewDataLiteral<ui64>(7ULL); + const auto pgmReturn = pb.Filter(stream, limit, + [&](TRuntimeNode item) { + return pb.AggrEquals(pb.CountBits(item), pb.NewDataLiteral<ui64>(3ULL)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 7ULL); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 11ULL); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 13ULL); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 14ULL); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 19ULL); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 21ULL); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 22ULL); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterWithLimitOverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto type = pb.NewFlowType(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<ui32>::Id))); + const auto flow = pb.ChainMap(pb.SourceOf(type), pb.NewDataLiteral<ui64>(0ULL), + [&](TRuntimeNode, TRuntimeNode state) -> TRuntimeNodePair { + return {state, pb.Increment(state)}; + } + ); + + const auto limit = pb.NewDataLiteral<ui64>(7ULL); + const auto pgmReturn = pb.FromFlow(pb.Filter(flow, limit, + [&](TRuntimeNode item) { + return pb.AggrEquals(pb.CountBits(item), pb.NewDataLiteral<ui64>(3ULL)); + })); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 7ULL); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 11ULL); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 13ULL); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 14ULL); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 19ULL); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 21ULL); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 22ULL); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_flatmap_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_flatmap_ut.cpp new file mode 100644 index 00000000000..8ad155473d5 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_flatmap_ut.cpp @@ -0,0 +1,853 @@ +#include "mkql_computation_node_ut.h" +#include <ydb/library/yql/minikql/mkql_runtime_version.h> + +namespace NKikimr { +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLFlatMapTest) { + Y_UNIT_TEST_LLVM(TestOverListAndPartialLists) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto list = pb.NewList(dataType, {data1, data2}); + const auto pgmReturn = pb.FlatMap(list, + [&](TRuntimeNode item) { + return pb.NewList(dataType, {pb.Add(item, data1), pb.Mul(item, data2)}); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestOverListAndStreams) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<i8>(3); + const auto data2 = pb.NewDataLiteral<i8>(-7); + const auto dataType = pb.NewDataType(NUdf::TDataType<i8>::Id); + const auto list = pb.NewList(dataType, {data1, data2}); + const auto pgmReturn = pb.FlatMap(list, + [&](TRuntimeNode item) { + return pb.Iterator(pb.NewList(dataType, {pb.Plus(item), pb.Minus(item)}), {}); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), -3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), -7); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), 7); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestOverStreamAndPartialLists) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui16>(10); + const auto data2 = pb.NewDataLiteral<ui16>(20); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id); + const auto list = pb.NewList(dataType, {data1, data2}); + const auto pgmReturn = pb.FlatMap(pb.Iterator(list, {}), + [&](TRuntimeNode item) { + return pb.NewList(dataType, {pb.Sub(item, data1), pb.Unwrap(pb.Div(item, data2), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0)}); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 10); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 1); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestOverFlowAndPartialLists) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui16>(10); + const auto data2 = pb.NewDataLiteral<ui16>(20); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id); + const auto list = pb.NewList(dataType, {data1, data2}); + const auto pgmReturn = pb.FromFlow(pb.FlatMap(pb.ToFlow(pb.Iterator(list, {})), + [&](TRuntimeNode item) { + return pb.NewList(dataType, {pb.Sub(item, data1), pb.Unwrap(pb.Div(item, data2), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0)}); + })); + + const auto& graph = setup.BuildGraph(pgmReturn); + const NUdf::TUnboxedValue& iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 10); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 1); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestOverStreamAndStreams) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data = pb.NewDataLiteral<i32>(-100); + const auto data0 = pb.NewDataLiteral<i32>(0); + const auto data1 = pb.NewDataLiteral<i32>(3); + const auto data2 = pb.NewDataLiteral<i32>(7); + const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2}); + const auto pgmReturn = pb.FlatMap(pb.Iterator(list, {}), + [&](TRuntimeNode item) { + return pb.Iterator(pb.NewList(pb.NewOptionalType(dataType), + {pb.Mod(data, item), pb.Div(data, item)}), {}); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -33); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -2); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -14); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestOverFlowAndStreams) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data = pb.NewDataLiteral<i32>(-100); + const auto data0 = pb.NewDataLiteral<i32>(0); + const auto data1 = pb.NewDataLiteral<i32>(3); + const auto data2 = pb.NewDataLiteral<i32>(7); + const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2}); + const auto pgmReturn = pb.FromFlow(pb.FlatMap(pb.ToFlow(list), + [&](TRuntimeNode item) { + return pb.Iterator(pb.NewList(pb.NewOptionalType(dataType), + {pb.Mod(data, item), pb.Div(data, item)}), {}); + })); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -33); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -2); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -14); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestOverFlowAndFlows) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data = pb.NewDataLiteral<i32>(-100); + const auto data0 = pb.NewDataLiteral<i32>(0); + const auto data1 = pb.NewDataLiteral<i32>(3); + const auto data2 = pb.NewDataLiteral<i32>(7); + const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2}); + const auto pgmReturn = pb.FromFlow(pb.FlatMap(pb.ToFlow(list), + [&](TRuntimeNode item) { + return pb.ToFlow(pb.NewList(pb.NewOptionalType(dataType), + {pb.Mod(data, item), pb.Div(data, item)})); + })); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -33); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -2); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -14); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestOverListAndFlows) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data = pb.NewDataLiteral<i32>(-100); + const auto data0 = pb.NewDataLiteral<i32>(0); + const auto data1 = pb.NewDataLiteral<i32>(3); + const auto data2 = pb.NewDataLiteral<i32>(7); + const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2}); + const auto pgmReturn = pb.FromFlow(pb.FlatMap(list, + [&](TRuntimeNode item) { + return pb.ToFlow(pb.NewList(pb.NewOptionalType(dataType), + {pb.Mod(data, item), pb.Div(data, item)})); + })); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -33); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -2); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -14); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestOverFlowAndIndependentFlows) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data = pb.NewDataLiteral<i32>(-100); + const auto data0 = pb.NewDataLiteral<i32>(0); + const auto data1 = pb.NewDataLiteral<i32>(3); + const auto data2 = pb.NewDataLiteral<i32>(7); + const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2}); + const auto pgmReturn = pb.FromFlow(pb.FlatMap(pb.ToFlow(list), + [&](TRuntimeNode) { + return pb.Map(pb.ToFlow(pb.NewList(dataType, {data, data})), [&](TRuntimeNode it) { return pb.Abs(it); }); + })); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestOverListAndIndependentFlows) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data = pb.NewDataLiteral<i32>(-100); + const auto data0 = pb.NewDataLiteral<i32>(0); + const auto data1 = pb.NewDataLiteral<i32>(3); + const auto data2 = pb.NewDataLiteral<i32>(7); + const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2}); + const auto pgmReturn = pb.FromFlow(pb.FlatMap(list, + [&](TRuntimeNode) { + return pb.Map(pb.ToFlow(pb.NewList(dataType, {data, data})), [&](TRuntimeNode it) { return pb.Minus(it); }); + })); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestOverFlowAndPartialOptionals) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data = pb.NewDataLiteral<i64>(-100); + const auto data0 = pb.NewDataLiteral<i64>(0); + const auto data1 = pb.NewDataLiteral<i64>(3); + const auto data2 = pb.NewDataLiteral<i64>(7); + const auto dataType = pb.NewDataType(NUdf::TDataType<i64>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2}); + const auto pgmReturn = pb.FromFlow(pb.FlatMap(pb.ToFlow(pb.Iterator(list, {})), + [&](TRuntimeNode item) { + return pb.Div(data, item); + })); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), -33); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), -14); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestOverStreamAndPartialOptionals) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data = pb.NewDataLiteral<i64>(-100); + const auto data0 = pb.NewDataLiteral<i64>(0); + const auto data1 = pb.NewDataLiteral<i64>(3); + const auto data2 = pb.NewDataLiteral<i64>(7); + const auto dataType = pb.NewDataType(NUdf::TDataType<i64>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2}); + const auto pgmReturn = pb.FlatMap(pb.Iterator(list, {}), + [&](TRuntimeNode item) { + return pb.Div(data, item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), -33); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), -14); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestOverListAndPartialOptionals) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<ui32>(0); + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2}); + const auto pgmReturn = pb.FlatMap(list, + [&](TRuntimeNode item) { + return pb.Div(data2, item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestOverListAndDoubleOptionals) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<ui32>(0); + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2}); + const auto pgmReturn = pb.FlatMap(list, + [&](TRuntimeNode item) { + return pb.NewOptional(pb.Div(data2, item)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestOverOptionalAndPartialOptionals) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto list = pb.NewOptional(data2); + const auto pgmReturn = pb.FlatMap(list, + [&](TRuntimeNode item) { + return pb.Div(item, data2); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto value = graph->GetValue(); + UNIT_ASSERT(value); + UNIT_ASSERT_VALUES_EQUAL(value.template Get<ui32>(), 1); + } + + Y_UNIT_TEST_LLVM(TestOverOptionalAndPartialLists) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto list = pb.NewOptional(data2); + const auto pgmReturn = pb.FlatMap(list, + [&](TRuntimeNode item) { + return pb.Append(pb.AsList(item), data1); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestOverListAndPartialListsLazy) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1U); + const auto data2 = pb.NewDataLiteral<ui32>(2U); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto list = pb.NewList(dataType, {data1, data2}); + + const auto pgmReturn = pb.FlatMap(pb.LazyList(list), + [&](TRuntimeNode item) { + return pb.NewList(dataType, {pb.Add(item, data1), pb.Mul(item, data2)}); + }); + + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestOverListAndPartialOptionalsLazy) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<ui32>(0U); + const auto data2 = pb.NewDataLiteral<ui32>(2U); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto list = pb.NewList(dataType, {data0, data2}); + + const auto pgmReturn = pb.FlatMap(pb.LazyList(list), + [&](TRuntimeNode item) { return pb.Div(data2, item); } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u + Y_UNIT_TEST_LLVM(TestNarrowWithList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3}); + + const auto pgmReturn = pb.Collect(pb.NarrowFlatMap(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U),pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.FlatMap(pb.NewList(dataType, items), [](TRuntimeNode item){ return item; }); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -3); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestNarrowWithFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3}); + + const auto pgmReturn = pb.Collect(pb.NarrowFlatMap(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U),pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.FlatMap(pb.ToFlow(pb.NewList(dataType, items)), [](TRuntimeNode item){ return item; }); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -3); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestNarrowWithIndependentFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + + const auto list = pb.NewList(tupleType, {data, data, data}); + + const auto pgmReturn = pb.Collect(pb.NarrowFlatMap(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U),pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList) { return pb.Map( + pb.ToFlow(pb.NewList(pb.NewDataType(NUdf::TDataType<float>::Id), {pb.NewDataLiteral<float>(+1.f), pb.NewDataLiteral<float>(-1.f)})), + [&](TRuntimeNode item) { return pb.Minus(item); }); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -1.f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), +1.f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -1.f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), +1.f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -1.f); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), +1.f); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestThinNarrowWithList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto tupleType = pb.NewTupleType({}); + + const auto data = pb.NewTuple(tupleType, {}); + const auto list = pb.NewList(tupleType, {data, data, data}); + + const auto pgmReturn = pb.Collect(pb.NarrowFlatMap(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode) -> TRuntimeNode::TList { return {}; }), + [&](TRuntimeNode::TList) -> TRuntimeNode { return pb.Replicate(pb.NewDataLiteral<i32>(7), pb.NewDataLiteral<ui64>(3), __FILE__, __LINE__, 0); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestOverFlowAndWideFlows) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data = pb.NewDataLiteral<i32>(-100); + const auto data0 = pb.NewDataLiteral<i32>(0); + const auto data1 = pb.NewDataLiteral<i32>(3); + const auto data2 = pb.NewDataLiteral<i32>(7); + const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2}); + const auto pgmReturn = pb.FromFlow(pb.NarrowMap(pb.FlatMap(pb.ToFlow(list), + [&](TRuntimeNode item) { + return pb.ExpandMap(pb.ToFlow(pb.NewList(pb.NewOptionalType(dataType), + {pb.Mod(data, item), pb.Div(data, item)})), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Plus(item), pb.Minus(item)}; }); + }), + [&](TRuntimeNode::TList items) { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +1); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -33); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +33); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -2); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +2); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -14); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +14); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestOverListAndWideFlows) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data = pb.NewDataLiteral<i32>(-100); + const auto data0 = pb.NewDataLiteral<i32>(0); + const auto data1 = pb.NewDataLiteral<i32>(3); + const auto data2 = pb.NewDataLiteral<i32>(7); + const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2}); + const auto pgmReturn = pb.FromFlow(pb.NarrowMap(pb.FlatMap(list, + [&](TRuntimeNode item) { + return pb.ExpandMap(pb.ToFlow(pb.NewList(pb.NewOptionalType(dataType), + {pb.Mod(data, item), pb.Div(data, item)})), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Minus(item), pb.Plus(item)}; }); + }), + [&](TRuntimeNode::TList items) { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -1); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +33); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -33); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +2); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -2); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +14); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -14); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestOverFlowAndIndependentWideFlows) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data = pb.NewDataLiteral<i32>(-100); + const auto data0 = pb.NewDataLiteral<i32>(0); + const auto data1 = pb.NewDataLiteral<i32>(3); + const auto data2 = pb.NewDataLiteral<i32>(7); + const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2}); + const auto pgmReturn = pb.FromFlow(pb.NarrowMap(pb.FlatMap(pb.ToFlow(list), + [&](TRuntimeNode) { + return pb.ExpandMap(pb.ToFlow(pb.NewList(dataType, {data, data})), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Plus(item), pb.Minus(item)}; }); + }), + [&](TRuntimeNode::TList items) { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -100); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -100); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -100); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -100); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -100); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -100); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestOverListAndIndependentWideFlows) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data = pb.NewDataLiteral<i32>(-100); + const auto data0 = pb.NewDataLiteral<i32>(0); + const auto data1 = pb.NewDataLiteral<i32>(3); + const auto data2 = pb.NewDataLiteral<i32>(7); + const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2}); + const auto pgmReturn = pb.FromFlow(pb.NarrowMap(pb.FlatMap(list, + [&](TRuntimeNode) { + return pb.ExpandMap(pb.ToFlow(pb.NewList(dataType, {data, data})), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Minus(item), pb.Plus(item)}; }); + }), + [&](TRuntimeNode::TList items) { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +100); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +100); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +100); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +100); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +100); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +100); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -100); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } +#endif +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_fold_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_fold_ut.cpp new file mode 100644 index 00000000000..951aa0158b5 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_fold_ut.cpp @@ -0,0 +1,1207 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> + +#include <random> +#include <ctime> +#include <algorithm> + +namespace NKikimr { +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLFoldNodeTest) { + Y_UNIT_TEST_LLVM(TestFoldOverList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto data1 = pb.NewDataLiteral<ui32>(1); + auto data2 = pb.NewDataLiteral<ui32>(2); + auto data3 = pb.NewDataLiteral<ui32>(3); + auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + auto list = pb.NewList(dataType, {data1, data2, data3}); + auto pgmReturn = pb.Fold(list, pb.NewDataLiteral<ui32>(0), + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.Add(item, state); + }); + + auto graph = setup.BuildGraph(pgmReturn); + auto res = graph->GetValue().template Get<ui32>(); + UNIT_ASSERT_VALUES_EQUAL(res, 6); + } + + Y_UNIT_TEST_LLVM(TestFold1OverEmptyList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + auto list = pb.NewEmptyList(dataType); + auto data2 = pb.NewDataLiteral<ui32>(2); + auto pgmReturn = pb.Fold1(list, [&](TRuntimeNode item) { + return pb.Mul(item, data2); + }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.Add(item, state); + }); + + auto graph = setup.BuildGraph(pgmReturn); + auto value = graph->GetValue(); + UNIT_ASSERT(!value); + } + + Y_UNIT_TEST_LLVM(TestFold1OverSingleElementList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + auto data1 = pb.NewDataLiteral<ui32>(1); + auto data2 = pb.NewDataLiteral<ui32>(2); + auto list = pb.NewList(dataType, {data1}); + auto pgmReturn = pb.Fold1(list, + [&](TRuntimeNode item) { + return pb.Mul(item, data2); + }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.Add(item, state); + }); + + auto graph = setup.BuildGraph(pgmReturn); + auto value = graph->GetValue(); + UNIT_ASSERT(value); + UNIT_ASSERT_VALUES_EQUAL(value.template Get<ui32>(), 2); + } + + Y_UNIT_TEST_LLVM(TestFold1OverManyElementList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + auto data1 = pb.NewDataLiteral<ui32>(1); + auto data2 = pb.NewDataLiteral<ui32>(2); + auto list = pb.NewList(dataType, {data1, data2}); + auto pgmReturn = pb.Fold1(list, + [&](TRuntimeNode item) { + return pb.Mul(item, data2); + }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.Add(item, state); + }); + + auto graph = setup.BuildGraph(pgmReturn); + auto value = graph->GetValue(); + UNIT_ASSERT(value); + UNIT_ASSERT_VALUES_EQUAL(value.template Get<ui32>(), 4); + } + + Y_UNIT_TEST_LLVM(TestFoldWithAggrAdd) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<float>::Id)); + auto data1 = pb.NewOptional(pb.NewDataLiteral<float>(1)); + auto data2 = pb.NewOptional(pb.NewDataLiteral<float>(2)); + auto data3 = pb.NewOptional(pb.NewDataLiteral<float>(3)); + auto data4 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<float>::Id); + auto data0 = pb.NewOptional(pb.NewDataLiteral<float>(42)); + auto list = pb.NewList(dataType, {data4, data3, data2, data1}); + + auto pgmReturn = pb.Fold(list, data0, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.AggrAdd(pb.Increment(item), pb.Decrement(state)); + }); + + auto graph = setup.BuildGraph(pgmReturn); + auto res = graph->GetValue().template Get<float>(); + UNIT_ASSERT_VALUES_EQUAL(res, 47); + } + + Y_UNIT_TEST_LLVM(TestNestedApply) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto data1 = pb.NewDataLiteral<i32>(1); + auto data2 = pb.NewDataLiteral<i32>(2); + auto data3 = pb.NewDataLiteral<i32>(3); + auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id); + auto list = pb.NewList(dataType, {data1, data2, data3}); + + const auto callType = TCallableTypeBuilder(pb.GetTypeEnvironment(), "TEST", dataType).Add(dataType).Add(dataType).Build(); + + auto pgmReturn = pb.Fold(list, pb.NewDataLiteral<i32>(100), + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.Apply(pb.Callable(callType, + [&](const TArrayRef<const TRuntimeNode>& args) { + return pb.Sub(args[1], args[0]); + }), {item, state}); + }); + + auto graph = setup.BuildGraph(pgmReturn); + auto res = graph->GetValue().template Get<i32>(); + UNIT_ASSERT_VALUES_EQUAL(res, 94); + } + + Y_UNIT_TEST_LLVM(TestLogicalOpts) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto truth = pb.NewDataLiteral(true); + auto falsehood = pb.NewDataLiteral(false); + auto type = pb.NewDataType(NUdf::TDataType<bool>::Id); + auto args = pb.NewList(type, {truth, falsehood}); + + auto optTruth = pb.NewOptional(truth); + auto optFalsehood = pb.NewOptional(falsehood); + auto empty = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id); + auto optType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<bool>::Id)); + auto opts = pb.NewList(optType, {empty, optTruth, optFalsehood}); + + auto pgmReturn = pb.Fold(opts, pb.NewEmptyList(optType), + [&](TRuntimeNode item, TRuntimeNode state) { + const auto append = pb.Append(state, pb.Not(item)); + + const auto one = pb.Fold(args, pb.NewEmptyList(optType), + [&](TRuntimeNode item2, TRuntimeNode state2) { + state2 = pb.Append(state2, pb.And({item, item2})); + state2 = pb.Append(state2, pb.And({item2, item})); + state2 = pb.Append(state2, pb.Or({item, item2})); + state2 = pb.Append(state2, pb.Or({item2, item})); + state2 = pb.Append(state2, pb.Xor({item, item2})); + state2 = pb.Append(state2, pb.Xor({item2, item})); + return state2; + }); + + const auto two = pb.Fold(opts, pb.NewEmptyList(optType), + [&](TRuntimeNode item2, TRuntimeNode state2) { + state2 = pb.Append(state2, pb.And({item, item2})); + state2 = pb.Append(state2, pb.Or({item, item2})); + state2 = pb.Append(state2, pb.Xor({item, item2})); + return state2; + }); + + return pb.Extend({append, one, two}); + }); + + auto graph = setup.BuildGraph(pgmReturn); + + auto res = graph->GetValue(); + UNIT_ASSERT_VALUES_EQUAL(res.GetListLength(), 66ULL); + auto iterator = res.GetListIterator(); + + NUdf::TUnboxedValue item; + + /// empty + // not + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // and + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // or + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + + // xor + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // and + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + + // or + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // xor + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // and + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // or + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // xor + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // and + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // or + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + + // xor + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // and + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + + // or + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // xor + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + + /// true + // not + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + + // and + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + + // or + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + + // xor + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + + // and + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + + // or + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + + // xor + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + + // and + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // or + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + + // xor + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // and + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + + // or + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + + // xor + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + + // and + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + + // or + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + + // xor + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + + /// false + // not + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + + // and + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + + // or + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + + // xor + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + + // and + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + + // or + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + + // xor + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + + // and + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + + // or + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // xor + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + + // and + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + + // or + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + + // xor + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + + // and + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + + // or + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + + // xor + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFoldWithListInState) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto data1 = pb.NewDataLiteral<ui32>(1); + auto data2 = pb.NewDataLiteral<ui32>(2); + auto data3 = pb.NewDataLiteral<ui32>(3); + auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + auto list = pb.NewList(dataType, {data1, data2, data3}); + auto optType = pb.NewOptionalType(dataType); + auto empty = pb.AddMember(pb.AddMember( + pb.NewEmptyStruct(), "Max", pb.NewEmptyOptional(optType)), + "List", pb.NewEmptyList(dataType)); + + auto pgmReturn = pb.Fold(list, empty, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Max", + pb.IfPresent({pb.Member(state, "Max")}, + [&](TRuntimeNode::TList oldMax) { + return pb.NewOptional(pb.Max(oldMax.front(), item)); + }, pb.NewOptional(item))), + "List", pb.Append(pb.Member(state, "List"), item) + ); + }); + + auto graph = setup.BuildGraph(pgmReturn); + auto iterator = graph->GetValue().GetElement(0).GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetElement(1).template Get<ui32>(), 3); + } + + Y_UNIT_TEST_LLVM(TestManyAppend) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto zeroList = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id)); + zeroList = pb.Append(zeroList, pb.NewDataLiteral<ui32>(0)); + const ui32 n = 13; + for (ui32 i = 0; i < n; ++i) + zeroList = pb.Extend(zeroList, zeroList); + + auto state = pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Counter", + pb.NewDataLiteral<ui32>(0)), "NewList", + pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id))); + + auto fold = pb.Fold(zeroList, state, + [&](TRuntimeNode item, TRuntimeNode state) { + Y_UNUSED(item); + auto oldList = pb.Member(state, "NewList"); + auto oldCounter = pb.Member(state, "Counter"); + return pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Counter", + pb.Add(oldCounter, pb.NewDataLiteral<ui32>(1))), + "NewList", pb.Append(oldList, oldCounter)); + }); + + auto pgmReturn = pb.Member(fold, "NewList"); + + auto graph = setup.BuildGraph(pgmReturn); + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 1 << n); + + auto iterator = graph->GetValue().GetListIterator(); + ui32 i = 0; + for (NUdf::TUnboxedValue item; iterator.Next(item); ++i) { + UNIT_ASSERT_VALUES_EQUAL(i, item.template Get<ui32>()); + } + UNIT_ASSERT(!iterator.Skip()); + UNIT_ASSERT_VALUES_EQUAL(i, 1 << n); + } + + Y_UNIT_TEST_LLVM(TestManyPrepend) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto zeroList = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id)); + zeroList = pb.Append(zeroList, pb.NewDataLiteral<ui32>(0)); + const ui32 n = 13; + for (ui32 i = 0; i < n; ++i) + zeroList = pb.Extend(zeroList, zeroList); + + auto state = pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Counter", + pb.NewDataLiteral<ui32>(0)), "NewList", + pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id))); + + auto fold = pb.Fold(zeroList, state, + [&](TRuntimeNode item, TRuntimeNode state) { + Y_UNUSED(item); + auto oldList = pb.Member(state, "NewList"); + auto oldCounter = pb.Member(state, "Counter"); + return pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Counter", + pb.Add(oldCounter, pb.NewDataLiteral<ui32>(1))), + "NewList", pb.Prepend(oldCounter, oldList)); + }); + + auto pgmReturn = pb.Member(fold, "NewList"); + + auto graph = setup.BuildGraph(pgmReturn); + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 1 << n); + + auto iterator = graph->GetValue().GetListIterator(); + ui32 i = 1 << n; + for (NUdf::TUnboxedValue item; iterator.Next(item);) { + UNIT_ASSERT_VALUES_EQUAL(--i, item.template Get<ui32>()); + } + UNIT_ASSERT(!iterator.Skip()); + UNIT_ASSERT_VALUES_EQUAL(i, 0); + } + + Y_UNIT_TEST_LLVM(TestManyExtend) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto zeroList = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id)); + zeroList = pb.Append(zeroList, pb.NewDataLiteral<ui32>(0)); + const ui32 n = 13; + for (ui32 i = 0; i < n; ++i) + zeroList = pb.Extend(zeroList, zeroList); + + auto state = pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Counter", + pb.NewDataLiteral<ui32>(0)), "NewList", + pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id))); + + auto fold = pb.Fold(zeroList, state, + [&](TRuntimeNode item, TRuntimeNode state) { + Y_UNUSED(item); + auto oldList = pb.Member(state, "NewList"); + auto oldCounter = pb.Member(state, "Counter"); + auto oldCounterMul2 = pb.Mul(oldCounter, pb.NewDataLiteral<ui32>(2)); + auto extList = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id)); + extList = pb.Append(extList, oldCounterMul2); + extList = pb.Append(extList, pb.Increment(oldCounterMul2)); + return pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Counter", + pb.Add(oldCounter, pb.NewDataLiteral<ui32>(1))), + "NewList", pb.Extend(oldList, extList)); + }); + + auto pgmReturn = pb.Member(fold, "NewList"); + + auto graph = setup.BuildGraph(pgmReturn); + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 1 << (n+1)); + + auto iterator = graph->GetValue().GetListIterator(); + ui32 i = 0; + for (NUdf::TUnboxedValue item; iterator.Next(item); ++i) { + UNIT_ASSERT_VALUES_EQUAL(i, item.template Get<ui32>()); + } + UNIT_ASSERT(!iterator.Skip()); + UNIT_ASSERT_VALUES_EQUAL(i, 1 << (n + 1)); + } + + Y_UNIT_TEST_LLVM(TestFoldSingular) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto data1 = pb.NewDataLiteral<ui32>(1); + auto data2 = pb.NewDataLiteral<ui32>(2); + auto data3 = pb.NewDataLiteral<ui32>(3); + auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + auto list = pb.NewList(dataType, {data1, data2, data3}); + auto fold1 = pb.Fold(list, pb.NewDataLiteral<ui32>(0), + [&](TRuntimeNode item, TRuntimeNode state) { + Y_UNUSED(state); + return item; + }); + + auto fold2 = pb.Fold(list, pb.NewDataLiteral<ui32>(0), + [&](TRuntimeNode item, TRuntimeNode state) { + Y_UNUSED(item); + return state; + }); + + auto pgmReturn = pb.NewList(dataType, {fold1, fold2}); + + auto graph = setup.BuildGraph(pgmReturn); + auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestSumListSizes) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto itemType = pb.NewDataType(NUdf::TDataType<float>::Id); + auto item = pb.NewDataLiteral<float>(0.f); + + auto listType = pb.NewListType(itemType); + + auto data0 = pb.NewEmptyList(itemType); + auto data1 = pb.NewList(itemType, {item}); + auto data2 = pb.NewList(itemType, {item, item, item}); + auto data3 = pb.NewList(itemType, {item, item, item, item, item}); + + auto list = pb.NewList(listType, {data0, data1, data2, data3}); + + auto pgmReturn = pb.Fold1(list, + [&](TRuntimeNode item) { return pb.Length(item); }, + [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, pb.Length(item)); } + ); + + auto graph = setup.BuildGraph(pgmReturn); + UNIT_ASSERT_VALUES_EQUAL(9ULL, graph->GetValue().template Get<ui64>()); + } + + Y_UNIT_TEST_LLVM(TestHasListsItems) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto itemType = pb.NewDataType(NUdf::TDataType<float>::Id); + auto item = pb.NewDataLiteral<float>(0.f); + + auto listType = pb.NewListType(itemType); + + auto data0 = pb.NewEmptyList(itemType); + auto data1 = pb.NewList(itemType, {item}); + auto data2 = pb.NewEmptyList(itemType); + + auto list = pb.NewList(listType, {data0, data1, data2}); + + auto pgmReturn = pb.Fold(list, pb.NewOptional(pb.NewDataLiteral<bool>(false)), + [&](TRuntimeNode item, TRuntimeNode state) { return pb.Or({state, pb.HasItems(item)}); } + ); + + auto graph = setup.BuildGraph(pgmReturn); + UNIT_ASSERT(graph->GetValue().template Get<bool>()); + } + + Y_UNIT_TEST_LLVM(TestConcat) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("aa"); + auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("bbb"); + auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("zzzz"); + auto type = pb.NewDataType(NUdf::EDataSlot::String); + auto list = pb.NewList(type, {data1, data2, data3}); + auto pgmReturn = pb.Fold(list, data0, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.Concat(state, item); + }); + + auto graph = setup.BuildGraph(pgmReturn); + auto res = graph->GetValue(); + UNBOXED_VALUE_STR_EQUAL(res, "Xaabbbzzzz"); + } + + Y_UNIT_TEST_LLVM(TestConcatOpt) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + auto data0 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("")); + auto data1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("very large string")); + auto data2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>(" + ")); + auto data3 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("small")); + auto type = pb.NewOptionalType(pb.NewDataType(NUdf::EDataSlot::String)); + auto list = pb.NewList(type, {data1, data2, data3, data0}); + auto pgmReturn = pb.Fold(list, data0, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.Concat(state, item); + }); + + auto graph = setup.BuildGraph(pgmReturn); + auto res = graph->GetValue(); + UNBOXED_VALUE_STR_EQUAL(res, "very large string + small"); + } + + Y_UNIT_TEST_LLVM(TestAggrConcat) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto type = pb.NewOptionalType(pb.NewDataType(NUdf::EDataSlot::Utf8)); + const auto data0 = pb.NewEmptyOptional(type); + const auto data1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("PREFIX:")); + const auto data2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("very large string")); + const auto data3 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(":SUFFIX")); + const auto list = pb.NewList(type, {data0, data1, data0, data2, data3, data0}); + + const auto pgmReturn = pb.Fold1(list, + [&](TRuntimeNode item) { return item; }, + [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrConcat(state, item); } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto str = graph->GetValue(); + UNBOXED_VALUE_STR_EQUAL(str, "PREFIX:very large string:SUFFIX"); + } + + Y_UNIT_TEST_LLVM(TestLongFold) { + for (ui32 i = 0; i < 10; ++i) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const ui32 n = 1000; + + auto firstList = pb.Replicate(pb.NewDataLiteral<ui32>(0), + pb.NewDataLiteral<ui64>(n), "", 0, 0); + + auto secondList = pb.Replicate(firstList, pb.NewDataLiteral<ui64>(n), "", 0, 0); + + auto pgmReturn = pb.Fold(secondList, pb.NewDataLiteral<ui32>(0), + [&](TRuntimeNode item, TRuntimeNode state) { + auto partialSum = pb.Fold(item, pb.NewDataLiteral<ui32>(0), + [&](TRuntimeNode item, TRuntimeNode state) { + Y_UNUSED(item); + return pb.AggrAdd(state, pb.NewDataLiteral<ui32>(1)); + }); + + return pb.AggrAdd(state, partialSum); + }); + + + auto graph = setup.BuildGraph(pgmReturn); + auto value = graph->GetValue().template Get<ui32>(); + UNIT_ASSERT_VALUES_EQUAL(value, n * n); + } + } + + Y_UNIT_TEST_LLVM(TestFoldAggrAddIntervals) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto upper = i64(+1000LL); + const auto lower = i64(-1000LL); + const auto part = i64(100LL); + const auto from = pb.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&lower, sizeof(lower))); + const auto stop = pb.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&upper, sizeof(upper))); + const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&part, sizeof(part))); + const auto list = pb.ListFromRange(from, stop, step); + + const auto pgmReturn = pb.Fold1(pb.ListFromRange(from, stop, step), + [&](TRuntimeNode item) { return pb.NewOptional(item); }, + [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(pb.NewOptional(item), state); } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto value = graph->GetValue(); + UNIT_ASSERT_VALUES_EQUAL(value.template Get<i64>(), -1000LL); + } + + Y_UNIT_TEST_LLVM(TestFoldFoldPerf) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const ui32 n = 3333U; + + const auto firstList = pb.Replicate(pb.NewDataLiteral<ui32>(1), pb.NewDataLiteral<ui64>(n), "", 0, 0); + + const auto secondList = pb.Replicate(firstList, pb.NewDataLiteral<ui64>(n), "", 0, 0); + + const auto pgmReturn = pb.Fold(secondList, pb.NewDataLiteral<ui32>(0), + [&](TRuntimeNode item, TRuntimeNode state) { + const auto partialSum = pb.Fold(item, pb.NewDataLiteral<ui32>(0), + [&](TRuntimeNode i2, TRuntimeNode state) { + return pb.AggrAdd(state, i2); + }); + + return pb.AggrAdd(state, partialSum); + }); + + + const auto t1 = TInstant::Now(); + const auto graph = setup.BuildGraph(pgmReturn); + const auto t2 = TInstant::Now(); + const auto value = graph->GetValue().template Get<ui32>(); + const auto t3 = TInstant::Now(); + Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ")." << Endl; + UNIT_ASSERT_VALUES_EQUAL(value, n * n); + } + + std::vector<double> MakeSamples() { + std::default_random_engine eng; + std::uniform_real_distribution<double> unif(-999.0, +999.0); + + std::vector<double> samples(3333333U); + + eng.seed(std::time(nullptr)); + std::generate(samples.begin(), samples.end(), std::bind(std::move(unif), std::move(eng))); + return samples; + } + + static const auto Samples = MakeSamples(); + + Y_UNIT_TEST_LLVM(TestSumDoubleArrayListPerf) { + TSetup<LLVM> setup; + + const auto t = TInstant::Now(); + const double sum = std::accumulate(Samples.cbegin(), Samples.cend(), 0.0); + const auto cppTime = TInstant::Now() - t; + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Fold1(pb.Collect(TRuntimeNode(list, false)), + [&](TRuntimeNode item) { return item; }, + [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, item); } + ); + + const auto t1 = TInstant::Now(); + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items)); + std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>); + const auto t2 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t3 = TInstant::Now(); + Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl; + UNIT_ASSERT_VALUES_EQUAL(value.template Get<double>(), sum); + } + + Y_UNIT_TEST_LLVM(TestSumDoubleLazyListPerf) { + TSetup<LLVM> setup; + + const auto t = TInstant::Now(); + const double sum = std::accumulate(Samples.cbegin(), Samples.cend(), 0.0); + const auto cppTime = TInstant::Now() - t; + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Fold1(pb.LazyList(TRuntimeNode(list, false)), + [&](TRuntimeNode item) { return item; }, + [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, item); } + ); + + const auto t1 = TInstant::Now(); + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items)); + std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>); + const auto t2 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t3 = TInstant::Now(); + Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl; + UNIT_ASSERT_VALUES_EQUAL(value.template Get<double>(), sum); + } + + Y_UNIT_TEST_LLVM(TestSumDoubleFilteredArrayListPerf) { + TSetup<LLVM> setup; + + const auto t = TInstant::Now(); + const double sum = std::accumulate(Samples.cbegin(), Samples.cend(), 0.0, [](double s, double v) { return v > 0.0 ? s + v : s; }); + const auto cppTime = TInstant::Now() - t; + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Fold1( + pb.Filter(pb.Collect(TRuntimeNode(list, false)), + [&](TRuntimeNode item) { return pb.AggrGreater(item, pb.NewDataLiteral(0.0)); } + ), + [&](TRuntimeNode item) { return item; }, + [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, item); } + ); + + const auto t1 = TInstant::Now(); + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items)); + std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>); + const auto t2 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t3 = TInstant::Now(); + Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl; + UNIT_ASSERT_VALUES_EQUAL(value.template Get<double>(), sum); + } + + Y_UNIT_TEST_LLVM(TestSumDoubleFilteredLazyListPerf) { + TSetup<LLVM> setup; + + const auto t = TInstant::Now(); + const double sum = std::accumulate(Samples.cbegin(), Samples.cend(), 0.0, [](double s, double v) { return v > 0.0 ? s + v : s; }); + const auto cppTime = TInstant::Now() - t; + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Fold1( + pb.Filter(pb.LazyList(TRuntimeNode(list, false)), + [&](TRuntimeNode item) { return pb.AggrGreater(item, pb.NewDataLiteral(0.0)); } + ), + [&](TRuntimeNode item) { return item; }, + [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, item); } + ); + + const auto t1 = TInstant::Now(); + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items)); + std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>); + const auto t2 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t3 = TInstant::Now(); + Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl; + UNIT_ASSERT_VALUES_EQUAL(value.template Get<double>(), sum); + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleArrayListPerf) { + TSetup<LLVM> setup; + + double min(Samples.front()), max(Samples.front()), sum(0.0); + + const auto t = TInstant::Now(); + for (const auto v : Samples) { + min = std::fmin(min, v); + max = std::fmax(max, v); + sum += v; + } + const auto cppTime = TInstant::Now() - t; + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Fold1(pb.Collect(TRuntimeNode(list, false)), + [&](TRuntimeNode item) { return pb.NewTuple({item, item, item}); }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.NewTuple({pb.AggrMin(pb.Nth(state, 0U), item), pb.AggrMax(pb.Nth(state, 1U), item), pb.AggrAdd(pb.Nth(state, 2U), item)}); + }); + + const auto t1 = TInstant::Now(); + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items)); + std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>); + const auto t2 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t3 = TInstant::Now(); + Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl; + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(0U).template Get<double>(), min); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(1U).template Get<double>(), max); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(2U).template Get<double>(), sum); + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleLazyListPerf) { + TSetup<LLVM> setup; + + double min(Samples.front()), max(Samples.front()), sum(0.0); + + const auto t = TInstant::Now(); + for (const auto v : Samples) { + min = std::fmin(min, v); + max = std::fmax(max, v); + sum += v; + } + const auto cppTime = TInstant::Now() - t; + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Fold1(pb.LazyList(TRuntimeNode(list, false)), + [&](TRuntimeNode item) { return pb.NewTuple({item, item, item}); }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.NewTuple({pb.AggrMin(pb.Nth(state, 0U), item), pb.AggrMax(pb.Nth(state, 1U), item), pb.AggrAdd(pb.Nth(state, 2U), item)}); + }); + + const auto t1 = TInstant::Now(); + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items)); + std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>); + const auto t2 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t3 = TInstant::Now(); + Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl; + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(0U).template Get<double>(), min); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(1U).template Get<double>(), max); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(2U).template Get<double>(), sum); + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleFilteredArrayListPerf) { + TSetup<LLVM> setup; + + double min(std::nan("")), max(std::nan("")), sum(0.0); + + const auto t = TInstant::Now(); + for (const auto v : Samples) { + if (v < 0.0) { + min = std::fmin(min, v); + max = std::fmax(max, v); + sum += v; + } + } + const auto cppTime = TInstant::Now() - t; + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Fold1( + pb.Filter(pb.Collect(TRuntimeNode(list, false)), + [&](TRuntimeNode item) { return pb.AggrLess(item, pb.NewDataLiteral(0.0)); } + ), + [&](TRuntimeNode item) { return pb.NewTuple({item, item, item}); }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.NewTuple({pb.AggrMin(pb.Nth(state, 0U), item), pb.AggrMax(pb.Nth(state, 1U), item), pb.AggrAdd(pb.Nth(state, 2U), item)}); + }); + + const auto t1 = TInstant::Now(); + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items)); + std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>); + const auto t2 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t3 = TInstant::Now(); + Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl; + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(0U).template Get<double>(), min); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(1U).template Get<double>(), max); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(2U).template Get<double>(), sum); + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleFilteredLazyListPerf) { + TSetup<LLVM> setup; + + double min(std::nan("")), max(std::nan("")), sum(0.0); + + const auto t = TInstant::Now(); + for (const auto v : Samples) { + if (v < 0.0) { + min = std::fmin(min, v); + max = std::fmax(max, v); + sum += v; + } + } + const auto cppTime = TInstant::Now() - t; + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Fold1( + pb.Filter(pb.LazyList(TRuntimeNode(list, false)), + [&](TRuntimeNode item) { return pb.AggrLess(item, pb.NewDataLiteral(0.0)); } + ), + [&](TRuntimeNode item) { return pb.NewTuple({item, item, item}); }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.NewTuple({pb.AggrMin(pb.Nth(state, 0U), item), pb.AggrMax(pb.Nth(state, 1U), item), pb.AggrAdd(pb.Nth(state, 2U), item)}); + }); + + const auto t1 = TInstant::Now(); + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items)); + std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>); + const auto t2 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t3 = TInstant::Now(); + Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl; + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(0U).template Get<double>(), min); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(1U).template Get<double>(), max); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(2U).template Get<double>(), sum); + } + + Y_UNIT_TEST_LLVM(TestAvgDoubleByTupleFoldArrayListPerf) { + TSetup<LLVM> setup; + + const auto t = TInstant::Now(); + const double avg = std::accumulate(Samples.cbegin(), Samples.cend(), 0.0) / Samples.size(); + const auto cppTime = TInstant::Now() - t; + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto fold = pb.Fold(pb.Collect(TRuntimeNode(list, false)), + pb.NewTuple({pb.NewDataLiteral(0.0), pb.NewDataLiteral<ui64>(0ULL)}), + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0), item), pb.Increment(pb.Nth(state, 1))}); + }); + + const auto pgmReturn = pb.Div(pb.Nth(fold, 0U), pb.Nth(fold, 1U)); + + const auto t1 = TInstant::Now(); + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items)); + std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>); + const auto t2 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t3 = TInstant::Now(); + Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl; + UNIT_ASSERT_VALUES_EQUAL(value.template Get<double>(), avg); + } + + Y_UNIT_TEST_LLVM(TestAvgDoubleByTupleFoldLazyListPerf) { + TSetup<LLVM> setup; + + const auto t = TInstant::Now(); + const double avg = std::accumulate(Samples.cbegin(), Samples.cend(), 0.0) / Samples.size(); + const auto cppTime = TInstant::Now() - t; + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto fold = pb.Fold(pb.LazyList(TRuntimeNode(list, false)), + pb.NewTuple({pb.NewDataLiteral(0.0), pb.NewDataLiteral<ui64>(0ULL)}), + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0U), item), pb.Increment(pb.Nth(state, 1U))}); + }); + + const auto pgmReturn = pb.Div(pb.Nth(fold, 0U), pb.Nth(fold, 1U)); + + const auto t1 = TInstant::Now(); + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items)); + std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>); + const auto t2 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t3 = TInstant::Now(); + Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl; + UNIT_ASSERT_VALUES_EQUAL(value.template Get<double>(), avg); + } + + Y_UNIT_TEST_LLVM(TestAvgDoubleByCollectFoldLazyListPerf) { + TSetup<LLVM> setup; + + const auto t = TInstant::Now(); + const double avg = std::accumulate(Samples.cbegin(), Samples.cend(), 0.0) / Samples.size(); + const auto cppTime = TInstant::Now() - t; + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto src = pb.Collect(pb.LazyList(TRuntimeNode(list, false))); + const auto pgmReturn = pb.Div( + pb.Fold(src, pb.NewDataLiteral(0.0), + [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, item); } + ), + pb.Length(src) + ); + + const auto t1 = TInstant::Now(); + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items)); + std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>); + const auto t2 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t3 = TInstant::Now(); + Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl; + UNIT_ASSERT_VALUES_EQUAL(value.template Get<double>(), avg); + } +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_fromstring_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_fromstring_ut.cpp new file mode 100644 index 00000000000..1acb4460911 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_fromstring_ut.cpp @@ -0,0 +1,38 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> + +namespace NKikimr { +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLFromStringTest) { + Y_UNIT_TEST_LLVM(TestFromString) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto data = pb.NewDataLiteral<NUdf::EDataSlot::String>("abcdefg"); + const auto pgmReturn1 = pb.StrictFromString(data, pb.NewDecimalType(10, 7)); + const auto graph1 = setup.BuildGraph(pgmReturn1); + UNIT_ASSERT_EXCEPTION_CONTAINS(graph1->GetValue(), std::exception, R"(Terminate was called, reason(45): could not convert "abcdefg" to Decimal(10, 7))"); + + const auto pgmReturn2 = pb.StrictFromString(data, pb.NewDataType(NUdf::TDataType<ui64>::Id)); + const auto graph2 = setup.BuildGraph(pgmReturn2); + UNIT_ASSERT_EXCEPTION_CONTAINS(graph2->GetValue(), std::exception, R"(Terminate was called, reason(37): could not convert "abcdefg" to Uint64)"); + } + + Y_UNIT_TEST_LLVM(TestFromStringHugePayload) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + const auto data = pb.NewDataLiteral<NUdf::EDataSlot::String>(TString(10000, 'x')); + const auto pgmReturn1 = pb.StrictFromString(data, pb.NewDecimalType(10, 7)); + const auto graph1 = setup.BuildGraph(pgmReturn1); + UNIT_ASSERT_EXCEPTION_CONTAINS(graph1->GetValue(), std::exception, R"(Terminate was called, reason(5050): could not convert "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" (truncated) to Decimal(10, 7))"); + + const auto pgmReturn2 = pb.StrictFromString(data, pb.NewDataType(NUdf::TDataType<ui64>::Id)); + const auto graph2 = setup.BuildGraph(pgmReturn2); + UNIT_ASSERT_EXCEPTION_CONTAINS(graph2->GetValue(), std::exception, R"(Terminate was called, reason(5042): could not convert "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" (truncated) to Uint64)"); + } +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_grace_join_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_grace_join_ut.cpp new file mode 100644 index 00000000000..6f655ef78b1 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_grace_join_ut.cpp @@ -0,0 +1,2003 @@ +#include "mkql_computation_node_ut.h" +#include <ydb/library/yql/minikql/mkql_runtime_version.h> +#include <ydb/library/yql/minikql/comp_nodes/mkql_grace_join_imp.h> + + +#include <chrono> +#include <iostream> +#include <cstring> +#include <vector> +#include <cassert> +#include <cstdlib> +#include <stdlib.h> + +#include <util/system/compiler.h> +#include <util/stream/null.h> +#include <util/system/mem_info.h> + +#include <cstdint> + +namespace NKikimr { +namespace NMiniKQL { + +constexpr bool IsVerbose = false; +#define CTEST (IsVerbose ? Cerr : Cnull) + + +Y_UNIT_TEST_SUITE(TMiniKQLGraceJoinMemTest) { + Y_UNIT_TEST(TestMem1) { + + const ui64 TupleSize = 1024; + const ui64 NBuckets = 128; + const ui64 NTuples = 100000; + const ui64 BucketSize = (2* NTuples * (TupleSize + 1) ) / NBuckets; + + ui64 *bigTuple = (ui64 * ) malloc(TupleSize * sizeof(ui64)); + ui64 *buckets[NBuckets]; + ui64 tuplesPos[NBuckets]; + + for (ui64 i = 0; i < TupleSize; i++) + { + bigTuple[i] = std::rand() / (RAND_MAX / 10000); + } + + ui64 bucket = 0; + ui64 milliseconds = 0; + + const ui64 BitsForData = 30; + + char* a = (char * )malloc(1 << BitsForData); + char* b = (char *) malloc(1 << BitsForData); + UNIT_ASSERT(a); + UNIT_ASSERT(b); + + memset(a, 1, 1 << BitsForData); + memset(b, 2, 1 << BitsForData); + + std::chrono::steady_clock::time_point begin01 = std::chrono::steady_clock::now(); + + memcpy(b, a, 1 << BitsForData); + + std::chrono::steady_clock::time_point end01 = std::chrono::steady_clock::now(); + + UNIT_ASSERT(*a == 1); + UNIT_ASSERT(*b == 1); + + Y_DO_NOT_OPTIMIZE_AWAY(a); + Y_DO_NOT_OPTIMIZE_AWAY(b); + + ui64 microseconds = std::chrono::duration_cast<std::chrono::microseconds>(end01 - begin01).count(); + CTEST << "Time for memcpy = " << microseconds << "[microseconds]" << Endl; + CTEST << "Data size = " << (1<<BitsForData) / (1024 * 1024) << "[MB]" << Endl; + CTEST << "Memcpy speed = " << ( (1<<BitsForData) ) / (microseconds) << "MB/sec" << Endl; + CTEST << Endl; + + std::vector<std::vector<ui64>> vec_buckets; + vec_buckets.resize(NBuckets); + for (ui64 i = 0; i < NBuckets; i++) + { + vec_buckets[i].resize(2 * TupleSize * NTuples / (NBuckets - 1), 0); + vec_buckets[i].clear(); +// vec_buckets[i].reserve( 2 * TupleSize * NTuples / (NBuckets - 1)); + } + + for (ui64 i = 0; i < NBuckets; i++) { + buckets[i] = (ui64 * ) malloc( (BucketSize * sizeof(ui64) * 32) / 32); + memset( buckets[i], 1, (BucketSize * sizeof(ui64) * 32) / 32); + tuplesPos[i] = 0; + } + + + std::chrono::steady_clock::time_point begin02 = std::chrono::steady_clock::now(); + + for (ui64 i = 0; i < NTuples; i++) + { + bucket = i % NBuckets; +// bucket = std::rand() / ( RAND_MAX / (NBuckets-1)); + std::vector<ui64> &curr_vec = vec_buckets[bucket]; + curr_vec.insert(curr_vec.end(), bigTuple, bigTuple + TupleSize); + } + + std::chrono::steady_clock::time_point end02 = std::chrono::steady_clock::now(); + + milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(end02 - begin02).count(); + CTEST << "Time for std::insert = " << milliseconds << "[ms]" << Endl; + CTEST << "Total MB = " << (TupleSize * NTuples * sizeof(ui64) / (1024 * 1024)) << Endl; + CTEST << "std::insert speed = " << (TupleSize * NTuples * sizeof(ui64) * 1000) / (milliseconds * 1024 * 1024) << "MB/sec" << Endl; + CTEST << Endl; + + std::chrono::steady_clock::time_point begin03 = std::chrono::steady_clock::now(); + + for (ui64 i = 0; i < NTuples; i++) + { + + bucket = i % NBuckets; +// bucket = std::rand() / ( RAND_MAX / (NBuckets-1)); + + ui64 * dst = buckets[bucket] + tuplesPos[bucket]; + std::memcpy(dst, bigTuple, TupleSize*sizeof(ui64)); + tuplesPos[bucket] += TupleSize; + } + + std::chrono::steady_clock::time_point end03 = std::chrono::steady_clock::now(); + + milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(end03 - begin03).count(); + CTEST << "Time for std::memcpy = " << milliseconds << "[ms]" << Endl; + CTEST << "Total MB = " << (TupleSize * NTuples * sizeof(ui64) / (1024 * 1024)) << Endl; + CTEST << "std:memcpy speed = " << (TupleSize * NTuples * sizeof(ui64) * 1000) / (milliseconds * 1024 * 1024) << "MB/sec" << Endl; + CTEST << Endl; + + for (ui64 i = 0; i < NBuckets; i++) { + tuplesPos[i] = 0; + } + + + std::chrono::steady_clock::time_point begin04 = std::chrono::steady_clock::now(); + + for (ui64 i = 0; i < NTuples; i++) + { + bucket = std::rand() / ( RAND_MAX / (NBuckets-1)); + + ui64 * dst = buckets[bucket] + tuplesPos[bucket]; + + ui64 *dst1 = dst + 1; + ui64 *dst2 = dst + 2; + ui64 *dst3 = dst + 3; + ui64 *src = bigTuple; + ui64 *src1 = bigTuple + 1; + ui64 *src2 = bigTuple + 2; + ui64 *src3 = bigTuple + 3; + + for (ui64 i = 0; i < TupleSize; i += 4) + { + *dst++ = *src++; + *dst1++ = *src1++; + *dst2++ = *src2++; + *dst3++ = *src3++; + + } + tuplesPos[bucket] += TupleSize; + } + + std::chrono::steady_clock::time_point end04 = std::chrono::steady_clock::now(); + + milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(end04 - begin04).count(); + CTEST << "Time for loop copy = " << milliseconds << "[ms]" << Endl; + CTEST << "Total MB = " << (TupleSize * NTuples * sizeof(ui64) / (1024 * 1024)) << Endl; + CTEST << "Loop copy speed = " << (TupleSize * NTuples * sizeof(ui64) * 1000) / (milliseconds * 1024 * 1024) << "MB/sec" << Endl; + CTEST << Endl; + + for (ui64 i = 0; i < NBuckets; i++) { + free(buckets[i]); + } + + free(b); + free(a); + free(bigTuple); + + + UNIT_ASSERT(true); + + } + +} + + +Y_UNIT_TEST_SUITE(TMiniKQLGraceJoinImpTest) { + Y_UNIT_TEST_LLVM(TestImp1) { + TSetup<LLVM> setup; + ui64 tuple[11] = {0,1,2,3,4,5,6,7,8,9,10}; + ui32 strSizes[2] = {4, 4}; + char * strVals[] = {(char *)"aaaaa", (char *)"bbbb"}; + + char * bigStrVal[] = {(char *)"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + (char *)"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}; + ui32 bigStrSize[2] = {151, 151}; + + + NMemInfo::TMemInfo mi = NMemInfo::GetMemInfo(); + CTEST << "Mem usage before tables tuples added (MB): " << mi.RSS / (1024 * 1024) << Endl; + + GraceJoin::TTable bigTable(1,1,1,1); + GraceJoin::TTable smallTable(1,1,1,1); + GraceJoin::TTable joinTable(1,1,1,1); + + std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); + + const ui64 TupleSize = 1024; + + ui64 bigTuple[TupleSize]; + + for (ui64 i = 0; i < TupleSize; i++) { + bigTuple[i] = std::rand() / ( RAND_MAX / 10000 ); + } + + ui64 milliseconds = 0; + + + + const ui64 BigTableTuples = 600000; + const ui64 SmallTableTuples = 150000; + const ui64 BigTupleSize = 40; + + std::chrono::steady_clock::time_point begin03 = std::chrono::steady_clock::now(); + + + for ( ui64 i = 0; i < BigTableTuples; i++) { + tuple[1] = std::rand() % SmallTableTuples; + tuple[2] = tuple[1]; + bigTable.AddTuple(tuple, strVals, strSizes); + } + + smallTable.AddTuple(tuple, bigStrVal, bigStrSize); + + for ( ui64 i = 0; i < SmallTableTuples + 1; i++) { + tuple[1] = std::rand() % SmallTableTuples; + tuple[2] = tuple[1]; + smallTable.AddTuple(tuple, strVals, strSizes); + } + + std::chrono::steady_clock::time_point end03 = std::chrono::steady_clock::now(); + milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(end03 - begin03).count(); + CTEST << "Time for hash = " << milliseconds << "[ms]" << Endl; + CTEST << "Adding tuples speed: " << (BigTupleSize * (BigTableTuples + SmallTableTuples) * 1000) / ( milliseconds * 1024 * 1024) << "MB/sec" << Endl; + CTEST << Endl; + + mi = NMemInfo::GetMemInfo(); + CTEST << "Mem usage after tables tuples added (MB): " << mi.RSS / (1024 * 1024) << Endl; + + + bigTable.Clear(); + smallTable.Clear(); + + begin03 = std::chrono::steady_clock::now(); + + + for ( ui64 i = 0; i < BigTableTuples; i++) { + tuple[1] = std::rand() % SmallTableTuples; + tuple[2] = tuple[1]; + bigTable.AddTuple(tuple, strVals, strSizes); + } + + smallTable.AddTuple(tuple, bigStrVal, bigStrSize); + + for ( ui64 i = 0; i < SmallTableTuples + 1; i++) { + tuple[1] = std::rand() % SmallTableTuples; + tuple[2] = tuple[1]; + smallTable.AddTuple(tuple, strVals, strSizes); + } + + end03 = std::chrono::steady_clock::now(); + milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(end03 - begin03).count(); + CTEST << "Time for hash = " << milliseconds << "[ms]" << Endl; + CTEST << "Adding tuples speed: " << (BigTupleSize * (BigTableTuples + SmallTableTuples) * 1000) / ( milliseconds * 1024 * 1024) << "MB/sec" << Endl; + CTEST << Endl; + + mi = NMemInfo::GetMemInfo(); + CTEST << "Mem usage after tables tuples added (MB): " << mi.RSS / (1024 * 1024) << Endl; + + + std::vector<ui64> vals1, vals2; + std::vector<char *> strVals1, strVals2; + std::vector<ui32> strSizes1, strSizes2; + GraceJoin::TupleData td1, td2; + vals1.resize(100); + vals2.resize(100); + strVals1.resize(100); + strVals2.resize(100); + strSizes1.resize(100); + strSizes2.resize(100); + td1.IntColumns = vals1.data(); + td1.StrColumns = strVals1.data(); + td1.StrSizes = strSizes1.data(); + td2.IntColumns = vals2.data(); + td2.StrColumns = strVals2.data(); + td2.StrSizes = strSizes2.data(); + + ui64 numBigTuples = 0; + bigTable.ResetIterator(); + + std::chrono::steady_clock::time_point begin04 = std::chrono::steady_clock::now(); + + while(bigTable.NextTuple(td1)) { numBigTuples++; } + + CTEST << "Num of big tuples 1: " << numBigTuples << Endl; + + std::chrono::steady_clock::time_point end04 = std::chrono::steady_clock::now(); + CTEST << "Time for get 1 = " << std::chrono::duration_cast<std::chrono::milliseconds>(end04 - begin04).count() << "[ms]" << Endl; + CTEST << Endl; + + numBigTuples = 0; + bigTable.ResetIterator(); + + std::chrono::steady_clock::time_point begin041 = std::chrono::steady_clock::now(); + + while(bigTable.NextTuple(td2)) { numBigTuples++; } + + CTEST << "Num of big tuples 2: " << numBigTuples << Endl; + + std::chrono::steady_clock::time_point end041 = std::chrono::steady_clock::now(); + CTEST << "Time for get 2 = " << std::chrono::duration_cast<std::chrono::milliseconds>(end041 - begin041).count() << "[ms]" << Endl; + CTEST << Endl; + + + std::chrono::steady_clock::time_point begin05 = std::chrono::steady_clock::now(); + + joinTable.Join(smallTable,bigTable); + + std::chrono::steady_clock::time_point end05 = std::chrono::steady_clock::now(); + CTEST << "Time for join = " << std::chrono::duration_cast<std::chrono::milliseconds>(end05 - begin05).count() << "[ms]" << Endl; + CTEST << Endl; + + mi = NMemInfo::GetMemInfo(); + CTEST << "Mem usage after tables join (MB): " << mi.RSS / (1024 * 1024) << Endl; + + + joinTable.ResetIterator(); + ui64 numJoinedTuples = 0; + + + std::chrono::steady_clock::time_point begin042 = std::chrono::steady_clock::now(); + + while(joinTable.NextJoinedData(td1, td2)) { numJoinedTuples++; } + + CTEST << "Num of joined tuples : " << numJoinedTuples << Endl; + + std::chrono::steady_clock::time_point end042 = std::chrono::steady_clock::now(); + CTEST << "Time for get joined tuples: = " << std::chrono::duration_cast<std::chrono::milliseconds>(end042 - begin042).count() << "[ms]" << Endl; + CTEST << Endl; + + + std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now(); + CTEST << "Time difference = " << std::chrono::duration_cast<std::chrono::milliseconds>(end - begin).count() << "[ms]" << Endl; + CTEST << Endl; + + + } +} + +Y_UNIT_TEST_SUITE(TMiniKQLGraceJoinAnyTest) { + Y_UNIT_TEST_LLVM(TestImp2) { + TSetup<LLVM> setup; + ui64 tuple[11] = {0,1,2,3,4,5,6,7,8,9,10}; + ui32 strSizes[2] = {4, 4}; + char * strVals[] = {(char *)"aaaaa", (char *)"bbbb"}; + + char * bigStrVal[] = {(char *)"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + (char *)"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}; + ui32 bigStrSize[2] = {151, 151}; + + + + GraceJoin::TTable bigTable (1,1,1,1,0,0,1, nullptr, true); + GraceJoin::TTable smallTable(1,1,1,1,0,0,1, nullptr, true); + GraceJoin::TTable joinTable (1,1,1,1,0,0,1, nullptr, true); + + std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); + + const ui64 TupleSize = 1024; + + ui64 bigTuple[TupleSize]; + + for (ui64 i = 0; i < TupleSize; i++) { + bigTuple[i] = std::rand() / ( RAND_MAX / 10000 ); + } + + ui64 milliseconds = 0; + + + + const ui64 BigTableTuples = 600000; + const ui64 SmallTableTuples = 150000; + const ui64 BigTupleSize = 40; + + std::chrono::steady_clock::time_point begin03 = std::chrono::steady_clock::now(); + + + for ( ui64 i = 0; i < BigTableTuples; i++) { + tuple[1] = i % SmallTableTuples; + tuple[2] = tuple[1]; + bigTable.AddTuple(tuple, strVals, strSizes); + } + + smallTable.AddTuple(tuple, bigStrVal, bigStrSize); + + for ( ui64 i = 0; i < SmallTableTuples + 1; i++) { + tuple[1] = i; + tuple[2] = tuple[1]; + smallTable.AddTuple(tuple, strVals, strSizes); + } + + std::chrono::steady_clock::time_point end03 = std::chrono::steady_clock::now(); + milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(end03 - begin03).count(); + CTEST << "Time for hash = " << milliseconds << "[ms]" << Endl; + CTEST << "Adding tuples speed: " << (BigTupleSize * (BigTableTuples + SmallTableTuples) * 1000) / ( milliseconds * 1024 * 1024) << "MB/sec" << Endl; + CTEST << Endl; + + std::vector<ui64> vals1, vals2; + std::vector<char *> strVals1, strVals2; + std::vector<ui32> strSizes1, strSizes2; + GraceJoin::TupleData td1, td2; + vals1.resize(100); + vals2.resize(100); + strVals1.resize(100); + strVals2.resize(100); + strSizes1.resize(100); + strSizes2.resize(100); + td1.IntColumns = vals1.data(); + td1.StrColumns = strVals1.data(); + td1.StrSizes = strSizes1.data(); + td2.IntColumns = vals2.data(); + td2.StrColumns = strVals2.data(); + td2.StrSizes = strSizes2.data(); + + ui64 numBigTuples = 0; + bigTable.ResetIterator(); + + std::chrono::steady_clock::time_point begin04 = std::chrono::steady_clock::now(); + + while(bigTable.NextTuple(td1)) { numBigTuples++; } + + CTEST << "Num of big tuples 1: " << numBigTuples << Endl; + + std::chrono::steady_clock::time_point end04 = std::chrono::steady_clock::now(); + CTEST << "Time for get 1 = " << std::chrono::duration_cast<std::chrono::milliseconds>(end04 - begin04).count() << "[ms]" << Endl; + CTEST << Endl; + + numBigTuples = 0; + bigTable.ResetIterator(); + + std::chrono::steady_clock::time_point begin041 = std::chrono::steady_clock::now(); + + while(bigTable.NextTuple(td2)) { numBigTuples++; } + + CTEST << "Num of big tuples 2: " << numBigTuples << Endl; + + std::chrono::steady_clock::time_point end041 = std::chrono::steady_clock::now(); + CTEST << "Time for get 2 = " << std::chrono::duration_cast<std::chrono::milliseconds>(end041 - begin041).count() << "[ms]" << Endl; + CTEST << Endl; + + + std::chrono::steady_clock::time_point begin05 = std::chrono::steady_clock::now(); + + joinTable.Join(smallTable,bigTable); + + std::chrono::steady_clock::time_point end05 = std::chrono::steady_clock::now(); + CTEST << "Time for join = " << std::chrono::duration_cast<std::chrono::milliseconds>(end05 - begin05).count() << "[ms]" << Endl; + CTEST << Endl; + + joinTable.ResetIterator(); + ui64 numJoinedTuples = 0; + + + std::chrono::steady_clock::time_point begin042 = std::chrono::steady_clock::now(); + + while(joinTable.NextJoinedData(td1, td2)) { numJoinedTuples++; } + + CTEST << "Num of joined tuples : " << numJoinedTuples << Endl; + + std::chrono::steady_clock::time_point end042 = std::chrono::steady_clock::now(); + CTEST << "Time for get joined tuples: = " << std::chrono::duration_cast<std::chrono::milliseconds>(end042 - begin042).count() << "[ms]" << Endl; + CTEST << Endl; + + + std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now(); + CTEST << "Time difference = " << std::chrono::duration_cast<std::chrono::milliseconds>(end - begin).count() << "[ms]" << Endl; + CTEST << Endl; + + + } +} + +Y_UNIT_TEST_SUITE(TMiniKQLGraceSelfJoinTest) { + Y_UNIT_TEST_LLVM(TestImp3) { + TSetup<LLVM> setup; + ui64 tuple[11] = {0,1,2,3,4,5,6,7,8,9,10}; + ui32 strSizes[2] = {4, 4}; + char * strVals[] = {(char *)"aaaaa", (char *)"bbbb"}; + + char * bigStrVal[] = {(char *)"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + (char *)"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}; + ui32 bigStrSize[2] = {151, 151}; + + + + GraceJoin::TTable bigTable (1,1,1,1,0,0,1, nullptr, false); + GraceJoin::TTable smallTable(1,1,1,1,0,0,1, nullptr, false); + GraceJoin::TTable joinTable (1,1,1,1,0,0,1, nullptr, false); + + std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); + + const ui64 TupleSize = 1024; + + ui64 bigTuple[TupleSize]; + + for (ui64 i = 0; i < TupleSize; i++) { + bigTuple[i] = std::rand() / ( RAND_MAX / 10000 ); + } + + ui64 milliseconds = 0; + + + + const ui64 BigTableTuples = 600000; + const ui64 SmallTableTuples = 150000; + const ui64 BigTupleSize = 40; + + std::chrono::steady_clock::time_point begin03 = std::chrono::steady_clock::now(); + + + for ( ui64 i = 0; i < BigTableTuples; i++) { + tuple[1] = i % SmallTableTuples; + tuple[2] = tuple[1]; + bigTable.AddTuple(tuple, strVals, strSizes); + } + + smallTable.AddTuple(tuple, bigStrVal, bigStrSize); + + for ( ui64 i = 0; i < SmallTableTuples + 1; i++) { + tuple[1] = i; + tuple[2] = tuple[1]; + smallTable.AddTuple(tuple, strVals, strSizes); + } + + std::chrono::steady_clock::time_point end03 = std::chrono::steady_clock::now(); + milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(end03 - begin03).count(); + CTEST << "Time for hash = " << milliseconds << "[ms]" << Endl; + CTEST << "Adding tuples speed: " << (BigTupleSize * (BigTableTuples + SmallTableTuples) * 1000) / ( milliseconds * 1024 * 1024) << "MB/sec" << Endl; + CTEST << Endl; + + std::vector<ui64> vals1, vals2; + std::vector<char *> strVals1, strVals2; + std::vector<ui32> strSizes1, strSizes2; + GraceJoin::TupleData td1, td2; + vals1.resize(100); + vals2.resize(100); + strVals1.resize(100); + strVals2.resize(100); + strSizes1.resize(100); + strSizes2.resize(100); + td1.IntColumns = vals1.data(); + td1.StrColumns = strVals1.data(); + td1.StrSizes = strSizes1.data(); + td2.IntColumns = vals2.data(); + td2.StrColumns = strVals2.data(); + td2.StrSizes = strSizes2.data(); + + ui64 numBigTuples = 0; + bigTable.ResetIterator(); + + std::chrono::steady_clock::time_point begin04 = std::chrono::steady_clock::now(); + + while(bigTable.NextTuple(td1)) { numBigTuples++; } + + CTEST << "Num of big tuples 1: " << numBigTuples << Endl; + + std::chrono::steady_clock::time_point end04 = std::chrono::steady_clock::now(); + CTEST << "Time for get 1 = " << std::chrono::duration_cast<std::chrono::milliseconds>(end04 - begin04).count() << "[ms]" << Endl; + CTEST << Endl; + + numBigTuples = 0; + bigTable.ResetIterator(); + + std::chrono::steady_clock::time_point begin041 = std::chrono::steady_clock::now(); + + while(bigTable.NextTuple(td2)) { numBigTuples++; } + + CTEST << "Num of big tuples 2: " << numBigTuples << Endl; + + std::chrono::steady_clock::time_point end041 = std::chrono::steady_clock::now(); + CTEST << "Time for get 2 = " << std::chrono::duration_cast<std::chrono::milliseconds>(end041 - begin041).count() << "[ms]" << Endl; + CTEST << Endl; + + + std::chrono::steady_clock::time_point begin05 = std::chrono::steady_clock::now(); + + joinTable.Join(bigTable,bigTable); + + std::chrono::steady_clock::time_point end05 = std::chrono::steady_clock::now(); + CTEST << "Time for join = " << std::chrono::duration_cast<std::chrono::milliseconds>(end05 - begin05).count() << "[ms]" << Endl; + CTEST << Endl; + + joinTable.ResetIterator(); + ui64 numJoinedTuples = 0; + + + std::chrono::steady_clock::time_point begin042 = std::chrono::steady_clock::now(); + + while(joinTable.NextJoinedData(td1, td2)) { numJoinedTuples++; } + + CTEST << "Num of joined tuples : " << numJoinedTuples << Endl; + + std::chrono::steady_clock::time_point end042 = std::chrono::steady_clock::now(); + CTEST << "Time for get joined tuples: = " << std::chrono::duration_cast<std::chrono::milliseconds>(end042 - begin042).count() << "[ms]" << Endl; + CTEST << Endl; + + + std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now(); + CTEST << "Time difference = " << std::chrono::duration_cast<std::chrono::milliseconds>(end - begin).count() << "[ms]" << Endl; + CTEST << Endl; + + + } +} + +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 40u +Y_UNIT_TEST_SUITE(TMiniKQLSelfJoinTest) { + + Y_UNIT_TEST_LLVM(TestInner1) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(4); + const auto key4 = pb.NewDataLiteral<ui32>(4); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}), + pb.NewTuple({key4, payload4}) + }); + + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + })); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.SelfJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + EJoinKind::Inner, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "C"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "X"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "C"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "X"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "B"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "A"); + UNIT_ASSERT(!iterator.Next(tuple)); + + } + + + } + + Y_UNIT_TEST_LLVM(TestDiffKeys) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(4); + const auto key4 = pb.NewDataLiteral<ui32>(4); + const auto key11 = pb.NewDataLiteral<ui32>(1); + const auto key21 = pb.NewDataLiteral<ui32>(1); + const auto key31 = pb.NewDataLiteral<ui32>(2); + const auto key41 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, key11, payload1}), + pb.NewTuple({key2, key21, payload2}), + pb.NewTuple({key3, key31, payload3}), + pb.NewTuple({key4, key41, payload4}) + }); + + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + })); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.SelfJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + EJoinKind::Inner, {0U}, {1U}, {2U, 0U}, {2U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "C"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "A"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "B"); + UNIT_ASSERT(!iterator.Next(tuple)); + + } + + + } + + +} +#endif + +Y_UNIT_TEST_SUITE(TMiniKQLGraceJoinTest) { + + Y_UNIT_TEST_LLVM(TestInner1) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(4); + const auto key4 = pb.NewDataLiteral<ui32>(4); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + })); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + EJoinKind::Inner, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Z"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(!iterator.Next(tuple)); + + } + + + } + + Y_UNIT_TEST_LLVM(TestInnerDoubleCondition1) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(4); + const auto key4 = pb.NewDataLiteral<ui32>(4); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType1 = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto tupleType2 = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + + const auto list1 = pb.NewList(tupleType1, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType2, { + pb.NewTuple({key2, key2, payload4}), + pb.NewTuple({key3, key2, payload5}), + pb.NewTuple({key4, key1, payload6}) + }); + + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + })); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + EJoinKind::Inner, {0U, 0U}, {0U, 1U}, {1U, 0U}, {2U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(!iterator.Next(tuple)); + + } + + + } + + Y_UNIT_TEST_LLVM(TestInnerStringKey1) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("1"); + const auto key2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("2"); + const auto key3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("4"); + const auto key4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("4"); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + })); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + EJoinKind::Inner, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Z"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(!iterator.Next(tuple)); + + } + + + } + + + + Y_UNIT_TEST_LLVM(TMiniKQLGraceJoinTestInnerMulti1) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + })); + + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + EJoinKind::Inner, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestLeft1) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(3); + const auto key4 = pb.NewDataLiteral<ui32>(4); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + })); + + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + EJoinKind::Left, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + + for (ui32 i = 0; i < 3; i++) { + iterator.Next(tuple); + const auto cell = (tuple.GetElement(0)); + if (cell.AsStringRef() == "A") { + UNIT_ASSERT(!tuple.GetElement(1)); + } + if (cell.AsStringRef() == "B") { + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + } + if (cell.AsStringRef() == "C") { + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + } + } + + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + + } + } + + Y_UNIT_TEST_LLVM(TestLeftMulti1) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + })); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + EJoinKind::Left, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A"); + UNIT_ASSERT(!tuple.GetElement(1)); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestLeftSemi1) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<ui32>::Id) + })); + + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + EJoinKind::LeftSemi, {0U}, {0U}, {1U, 0U, 0U, 1U}, {}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestLeftOnly1) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto key5 = pb.NewDataLiteral<ui32>(4); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("D"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}), + pb.NewTuple({key4, payload4}), + pb.NewTuple({key5, payload4}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key2, payload5}), + pb.NewTuple({key3, payload6}), + pb.NewTuple({key4, payload7}) + }); + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<ui32>::Id) + })); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + EJoinKind::LeftOnly, {0U}, {0U}, {1U, 0U, 0U, 1U}, {}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "D"); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 4); + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A"); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 1); + + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestLeftSemiWithNullKey1) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key0 = pb.NewEmptyOptional(pb.NewDataType(NUdf::TDataType<ui32>::Id, true)); + const auto key1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1)); + const auto key2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2)); + const auto key3 = pb.NewOptional(pb.NewDataLiteral<ui32>(2)); + const auto key4 = pb.NewOptional(pb.NewDataLiteral<ui32>(3)); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id, true), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key0, payload4}), + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key0, payload3}), + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<ui32>::Id) + })); + + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + EJoinKind::LeftSemi, {0U}, {0U}, {1U, 0U, 0U, 1U}, {}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestLeftOnlyWithNullKey1) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key0 = pb.NewEmptyOptional(pb.NewDataType(NUdf::TDataType<ui32>::Id, true)); + const auto key1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1)); + const auto key2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2)); + const auto key3 = pb.NewOptional(pb.NewDataLiteral<ui32>(2)); + const auto key4 = pb.NewOptional(pb.NewDataLiteral<ui32>(3)); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id, true), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key0, payload4}), + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key0, payload3}), + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<ui32>::Id) + })); + + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + EJoinKind::LeftOnly, {0U}, {0U}, {1U, 0U, 0U, 1U}, {}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "X"); + UNIT_ASSERT(!tuple.GetElement(1)); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A"); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 1); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestRight1) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(3); + const auto key4 = pb.NewDataLiteral<ui32>(4); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + })); + + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + EJoinKind::Right, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + + for (ui32 i = 0; i < 3; i++) { + iterator.Next(tuple); + const auto cell = (tuple.GetElement(1)); + if (cell.AsStringRef() == "Z") { + UNIT_ASSERT(!tuple.GetElement(0)); + } + if (cell.AsStringRef() == "X") { + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + } + if (cell.AsStringRef() == "Y") { + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + } + } + + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + + } + } + + + Y_UNIT_TEST_LLVM(TestRightOnly1) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<ui32>::Id) + })); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + EJoinKind::RightOnly, {0U}, {0U}, {}, {1U, 0U, 0U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "Z"); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 3); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + + + Y_UNIT_TEST_LLVM(TestRightSemi1) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<ui32>::Id) + })); + + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + EJoinKind::RightSemi, {0U}, {0U}, {}, {1U, 0U, 0U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "X"); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "Y"); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + + Y_UNIT_TEST_LLVM(TestRightMulti1) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + })); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + EJoinKind::Right, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + + UNIT_ASSERT(iterator.Next(tuple)); + UNIT_ASSERT(!tuple.GetElement(0)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Z"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + + Y_UNIT_TEST_LLVM(TestRightSemiWithNullKey1) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key0 = pb.NewEmptyOptional(pb.NewDataType(NUdf::TDataType<ui32>::Id, true)); + const auto key1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1)); + const auto key2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2)); + const auto key3 = pb.NewOptional(pb.NewDataLiteral<ui32>(2)); + const auto key4 = pb.NewOptional(pb.NewDataLiteral<ui32>(3)); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id, true), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key0, payload4}), + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key0, payload3}), + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<ui32>::Id) + })); + + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + EJoinKind::RightSemi, {0U}, {0U}, {}, {1U, 0U, 0U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "X"); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "Y"); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestRightOnlyWithNullKey1) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key0 = pb.NewEmptyOptional(pb.NewDataType(NUdf::TDataType<ui32>::Id, true)); + const auto key1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1)); + const auto key2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2)); + const auto key3 = pb.NewOptional(pb.NewDataLiteral<ui32>(2)); + const auto key4 = pb.NewOptional(pb.NewDataLiteral<ui32>(3)); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id, true), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key0, payload4}), + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key0, payload3}), + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<ui32>::Id) + })); + + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + EJoinKind::RightOnly, {0U}, {0U}, {}, {1U, 0U, 0U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "Z"); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 3); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNIT_ASSERT(!tuple.GetElement(1)); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestFull1) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + })); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + EJoinKind::Full, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A"); + UNIT_ASSERT(!tuple.GetElement(1)); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Z"); + UNIT_ASSERT(!tuple.GetElement(0)); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + + Y_UNIT_TEST_LLVM(TestExclusion1) { + + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + })); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin( + pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + EJoinKind::Exclusion, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A"); + UNIT_ASSERT(!tuple.GetElement(1)); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Z"); + UNIT_ASSERT(!tuple.GetElement(0)); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + +} + + +} + +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp new file mode 100644 index 00000000000..1833bbf5095 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp @@ -0,0 +1,254 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + +template<bool UseLLVM> +TRuntimeNode MakeStream(TSetup<UseLLVM>& setup, ui64 count = 9U) { + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + TCallableBuilder callableBuilder(*setup.Env, "TestStream", + pgmBuilder.NewStreamType( + pgmBuilder.NewDataType(NUdf::EDataSlot::Uint64) + ) + ); + + callableBuilder.Add(pgmBuilder.NewDataLiteral(count)); + + return TRuntimeNode(callableBuilder.Build(), false); +} + +template<bool UseLLVM> +TRuntimeNode Group(TSetup<UseLLVM>& setup, TRuntimeNode stream, const std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)>& groupSwitch, + const std::function<TRuntimeNode(TRuntimeNode)>& handler = {}) +{ + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto keyExtractor = [&](TRuntimeNode item) { + return item; + }; + + stream = pgmBuilder.GroupingCore(stream, groupSwitch, keyExtractor, handler); + return pgmBuilder.FlatMap(stream, [&](TRuntimeNode grpItem) { + return pgmBuilder.Squeeze(pgmBuilder.Nth(grpItem, 1), + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("*"), + [&] (TRuntimeNode item, TRuntimeNode state) { + auto res = pgmBuilder.Concat(pgmBuilder.ToString(pgmBuilder.Nth(grpItem, 0)), pgmBuilder.ToString(item)); + res = pgmBuilder.Concat(state, res); + res = pgmBuilder.Concat(res, pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("*")); + return res; + }, + {}, {}); + }); +} + +template<bool UseLLVM> +TRuntimeNode GroupKeys(TSetup<UseLLVM>& setup, TRuntimeNode stream, const std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)>& groupSwitch) { + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto keyExtractor = [&](TRuntimeNode item) { + return item; + }; + + stream = pgmBuilder.GroupingCore(stream, groupSwitch, keyExtractor); + return pgmBuilder.Map(stream, [&](TRuntimeNode grpItem) { + return pgmBuilder.ToString(pgmBuilder.Nth(grpItem, 0)); + }); +} + +template<bool UseLLVM> +TRuntimeNode StreamToString(TSetup<UseLLVM>& setup, TRuntimeNode stream) { + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + return pgmBuilder.Squeeze(stream, pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("|"), [&] (TRuntimeNode item, TRuntimeNode state) { + return pgmBuilder.Concat(pgmBuilder.Concat(state, item), pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("|")); + }, {}, {}); +} + +} // unnamed + + +Y_UNIT_TEST_SUITE(TMiniKQLGroupingTest) { + Y_UNIT_TEST_LLVM(TestGrouping) { + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto stream = MakeStream(setup); + stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) { + Y_UNUSED(key); + return pgmBuilder.Equals(item, pgmBuilder.NewDataLiteral<ui64>(0)); + }); + auto pgm = StreamToString(setup, stream); + auto graph = setup.BuildGraph(pgm); + auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|"); + } + + Y_UNIT_TEST_LLVM(TestGroupingKeyNotEquals) { + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto stream = MakeStream(setup); + stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) { + return pgmBuilder.NotEquals(item, key); + }); + auto pgm = StreamToString(setup, stream); + auto graph = setup.BuildGraph(pgm); + auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*00*|*11*|*00*00*00*|*11*|*22*|*33*|"); + } + + Y_UNIT_TEST_LLVM(TestGroupingWithEmptyInput) { + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto stream = MakeStream(setup, 0); + stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) { + Y_UNUSED(key); + return pgmBuilder.Equals(item, pgmBuilder.NewDataLiteral<ui64>(0)); + }); + auto pgm = StreamToString(setup, stream); + auto graph = setup.BuildGraph(pgm); + auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|"); + } + + Y_UNIT_TEST_LLVM(TestSingleGroup) { + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto stream = MakeStream(setup); + stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) { + Y_UNUSED(key); + Y_UNUSED(item); + return pgmBuilder.NewDataLiteral<bool>(false); + }); + auto pgm = StreamToString(setup, stream); + auto graph = setup.BuildGraph(pgm); + auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*00*01*00*00*00*01*02*03*|"); + } + + Y_UNIT_TEST_LLVM(TestGroupingWithYield) { + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto stream = MakeStream(setup); + TSwitchInput switchInput; + switchInput.Indicies.push_back(0); + switchInput.InputType = stream.GetStaticType(); + + stream = pgmBuilder.Switch(stream, + MakeArrayRef(&switchInput, 1), + [&](ui32 /*index*/, TRuntimeNode item1) { + return Group(setup, item1, [&](TRuntimeNode key, TRuntimeNode item2) { + Y_UNUSED(key); + return pgmBuilder.Equals(item2, pgmBuilder.NewDataLiteral<ui64>(0)); + }); + }, + 1, + pgmBuilder.NewStreamType(pgmBuilder.NewDataType(NUdf::EDataSlot::String)) + ); + + auto pgm = StreamToString(setup, stream); + auto graph = setup.BuildGraph(pgm); + auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|"); + } + + Y_UNIT_TEST_LLVM(TestGroupingWithoutFetchingSubStreams) { + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto stream = MakeStream(setup); + + stream = GroupKeys(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) { + Y_UNUSED(key); + return pgmBuilder.Equals(item, pgmBuilder.NewDataLiteral<ui64>(0)); + }); + + auto pgm = StreamToString(setup, stream); + auto graph = setup.BuildGraph(pgm); + auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|"); + } + + Y_UNIT_TEST_LLVM(TestGroupingWithYieldAndWithoutFetchingSubStreams) { + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto stream = MakeStream(setup); + TSwitchInput switchInput; + switchInput.Indicies.push_back(0); + switchInput.InputType = stream.GetStaticType(); + + stream = pgmBuilder.Switch(stream, + MakeArrayRef(&switchInput, 1), + [&](ui32 /*index*/, TRuntimeNode item1) { + return GroupKeys(setup, item1, [&](TRuntimeNode key, TRuntimeNode item2) { + Y_UNUSED(key); + return pgmBuilder.Equals(item2, pgmBuilder.NewDataLiteral<ui64>(0)); + }); + }, + 1, + pgmBuilder.NewStreamType(pgmBuilder.NewDataType(NUdf::EDataSlot::String)) + ); + + auto pgm = StreamToString(setup, stream); + auto graph = setup.BuildGraph(pgm); + auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|"); + } + + Y_UNIT_TEST_LLVM(TestGroupingWithHandler) { + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto stream = MakeStream(setup); + stream = Group(setup, stream, + [&](TRuntimeNode key, TRuntimeNode item) { + Y_UNUSED(key); + return pgmBuilder.Equals(item, pgmBuilder.NewDataLiteral<ui64>(0)); + }, + [&](TRuntimeNode item) { + return pgmBuilder.Add(pgmBuilder.Convert(item, pgmBuilder.NewDataType(NUdf::EDataSlot::Int32)), pgmBuilder.NewDataLiteral<ui64>(1)); + } + ); + auto pgm = StreamToString(setup, stream); + auto graph = setup.BuildGraph(pgm); + auto streamVal = graph->GetValue(); + NUdf::TUnboxedValue result; + UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok); + + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*01*|*01*02*|*01*|*01*|*01*02*03*04*|"); + } +} + +} // NMiniKQL +} // NKikimr diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_heap_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_heap_ut.cpp new file mode 100644 index 00000000000..07c933affb5 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_heap_ut.cpp @@ -0,0 +1,319 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> + +#include <ydb/library/yql/utils/sort.h> + +namespace NKikimr { +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLHeapTest) { + Y_UNIT_TEST_LLVM(TestMakeHeap) { + const std::array<float, 10U> xxx = {{0.f, 13.f, -3.14f, 1212.f, -7898.8f, 21E4f, HUGE_VALF, -HUGE_VALF, 3673.f, -32764.f}}; + + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + std::array<TRuntimeNode, 10U> data; + std::transform(xxx.cbegin(), xxx.cend(), data.begin(), [&pb](float f) { return pb.NewDataLiteral(f); } ); + + const auto type = pb.NewDataType(NUdf::TDataType<float>::Id); + const auto list = pb.NewList(type, data); + + const auto pgmReturn = pb.MakeHeap(list, + [&](TRuntimeNode l, TRuntimeNode r) { + return pb.AggrLess(pb.Abs(l), pb.Abs(r)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto& result = graph->GetValue(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), xxx.size()); + + auto copy = xxx; + std::make_heap(copy.begin(), copy.end(), [](float l, float r){ return std::abs(l) < std::abs(r); }); + + for (auto i = 0U; i < copy.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL(copy[i], result.GetElement(i).template Get<float>()); + } + } + + Y_UNIT_TEST_LLVM(TestPopHeap) { + const std::array<double, 10U> xxx = {{0.0, 13.0, -3.140, 1212.0, -7898.8, 210000.0, 17E13, -HUGE_VAL, 3673.0, -32764.0}}; + + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + std::array<TRuntimeNode, 10U> data; + std::transform(xxx.cbegin(), xxx.cend(), data.begin(), [&pb](double f) { return pb.NewDataLiteral(f); } ); + + const auto type = pb.NewDataType(NUdf::TDataType<double>::Id); + const auto list = pb.NewList(type, data); + + const auto comparer = [&](TRuntimeNode l, TRuntimeNode r) { + return pb.AggrGreater(pb.Abs(l), pb.Abs(r)); + }; + + const auto pgmReturn = pb.PopHeap(pb.MakeHeap(list,comparer), comparer); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto& result = graph->GetValue(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), xxx.size()); + + auto copy = xxx; + const auto c = [](double l, double r){ return std::abs(l) > std::abs(r); }; + std::make_heap(copy.begin(), copy.end(), c); + std::pop_heap(copy.begin(), copy.end(), c); + + for (auto i = 0U; i < copy.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL(copy[i], result.GetElement(i).template Get<double>()); + } + } + + Y_UNIT_TEST_LLVM(TestSortHeap) { + const std::array<float, 10U> xxx = {{9E9f, -HUGE_VALF, 0.003f, 137.4f, -3.1415f, 1212.f, -7898.8f, 21E4f, 3673.f, -32764.f}}; + + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + std::array<TRuntimeNode, 10U> data; + std::transform(xxx.cbegin(), xxx.cend(), data.begin(), [&pb](float f) { return pb.NewDataLiteral(f); } ); + + const auto type = pb.NewDataType(NUdf::TDataType<float>::Id); + const auto list = pb.NewList(type, data); + + const auto pgmReturn = pb.SortHeap( + pb.MakeHeap(list, + [&](TRuntimeNode l, TRuntimeNode r) { + return pb.AggrGreater(l, r); + }), + [&](TRuntimeNode l, TRuntimeNode r) { + return pb.AggrGreater(l, r); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto& result = graph->GetValue(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), xxx.size()); + + auto copy = xxx; + std::make_heap(copy.begin(), copy.end(), std::greater<float>()); + std::sort_heap(copy.begin(), copy.end(), std::greater<float>()); + + for (auto i = 0U; i < copy.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL(copy[i], result.GetElement(i).template Get<float>()); + } + } + + Y_UNIT_TEST_LLVM(TestStableSort) { + const std::array<double, 10U> xxx = {{9E9f, -HUGE_VALF, 0.003f, HUGE_VALF, +3.1415f, -0.003f, -7898.8f, -3.1415f, 3673.f, 0.003f}}; + + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + std::array<TRuntimeNode, 10U> data; + std::transform(xxx.cbegin(), xxx.cend(), data.begin(), [&pb](double f) { return pb.NewDataLiteral(f); } ); + + const auto type = pb.NewDataType(NUdf::TDataType<double>::Id); + const auto list = pb.NewList(type, data); + + const auto pgmReturn = pb.StableSort(list, + [&](TRuntimeNode l, TRuntimeNode r) { + return pb.AggrGreater(pb.Abs(l), pb.Abs(r)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto& result = graph->GetValue(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), xxx.size()); + + auto copy = xxx; + std::stable_sort(copy.begin(), copy.end(), [](double l, double r){ return std::abs(l) > std::abs(r); }); + + for (auto i = 0U; i < copy.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL(copy[i], result.GetElement(i).template Get<double>()); + } + } + + Y_UNIT_TEST_LLVM(TestNthElement) { + const std::array<float, 10U> xxx = {{0.f, 13.f, -3.14f, 1212.f, -7898.8f, 21E4f, HUGE_VALF, -HUGE_VALF, 3673.f, -32764.f}}; + + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + std::array<TRuntimeNode, 10U> data; + std::transform(xxx.cbegin(), xxx.cend(), data.begin(), [&pb](float f) { return pb.NewDataLiteral(f); } ); + + const auto type = pb.NewDataType(NUdf::TDataType<float>::Id); + const auto list = pb.NewList(type, data); + const auto n = pb.NewDataLiteral<ui64>(4U); + + const auto pgmReturn = pb.NthElement(list, n, + [&](TRuntimeNode l, TRuntimeNode r) { + return pb.AggrGreater(pb.Abs(l), pb.Abs(r)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto& result = graph->GetValue(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), xxx.size()); + + auto copy = xxx; + NYql::FastNthElement(copy.begin(), copy.begin() + 4U, copy.end(), [](float l, float r){ return std::abs(l) > std::abs(r); }); + + for (auto i = 0U; i < copy.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL(copy[i], result.GetElement(i).template Get<float>()); + } + } + + Y_UNIT_TEST_LLVM(TestPartialSort) { + const std::array<double, 10U> xxx = {{0.0, 13.0, -3.14, 1212.0, -7898.8, 21.0E4, HUGE_VAL, -HUGE_VAL, 3673.0, -32764.0}}; + + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + std::array<TRuntimeNode, 10U> data; + std::transform(xxx.cbegin(), xxx.cend(), data.begin(), [&pb](double f) { return pb.NewDataLiteral(f); } ); + + const auto type = pb.NewDataType(NUdf::TDataType<double>::Id); + const auto list = pb.NewList(type, data); + const auto n = pb.NewDataLiteral<ui64>(6U); + + const auto pgmReturn = pb.PartialSort(list, n, + [&](TRuntimeNode l, TRuntimeNode r) { + return pb.AggrLess(pb.Abs(l), pb.Abs(r)); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto& result = graph->GetValue(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), xxx.size()); + + auto copy = xxx; + NYql::FastPartialSort(copy.begin(), copy.begin() + 6U, copy.end(), [](double l, double r){ return std::abs(l) < std::abs(r); }); + + for (auto i = 0U; i < copy.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL(copy[i], result.GetElement(i).template Get<double>()); + } + } + + Y_UNIT_TEST_LLVM(TestTopN) { + const std::array<double, 10U> xxx = {{0.0, 13.0, -3.140, -7898.8, 210000.0, 17E13, 1212.0, -HUGE_VAL, 3673.0, -32764.0}}; + + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + std::array<TRuntimeNode, 10U> data; + std::transform(xxx.cbegin(), xxx.cend(), data.begin(), [&pb](double f) { return pb.NewDataLiteral(f); } ); + + const auto type = pb.NewDataType(NUdf::TDataType<double>::Id); + const auto list = pb.NewList(type, data); + + const auto comparator = [&](TRuntimeNode l, TRuntimeNode r) { return pb.AggrGreater(pb.Abs(l), pb.Abs(r)); }; + + const auto n = 5ULL; + + const auto limit = pb.NewDataLiteral<ui64>(n); + const auto last = pb.Decrement(limit); + + const auto pgmReturn = pb.Take(pb.NthElement(pb.Fold(list, pb.NewEmptyList(type), + [&](TRuntimeNode item, TRuntimeNode state) { + const auto size = pb.Length(state); + + return pb.If(pb.AggrLess(size, limit), + pb.If(pb.AggrLess(size, last), + pb.Append(state, item), pb.MakeHeap(pb.Append(state, item), comparator)), + pb.If(comparator(item, pb.Unwrap(pb.ToOptional(state), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0)), + pb.PushHeap(pb.Append(pb.Take(pb.PopHeap(state, comparator), pb.Decrement(size)), item), comparator), + state + ) + ); + } + ), last, comparator), limit); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto& result = graph->GetValue(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), n); + + auto copy = xxx; + + const auto comp = [](double l, double r){ return std::abs(l) > std::abs(r); }; + NYql::FastNthElement(copy.begin(), copy.begin() + n - 1U, copy.end(), comp); + const auto mm = std::minmax_element(copy.begin(), copy.begin() + n, comp); + + double min = result.GetElement(0).template Get<double>(), max = min; + for (auto i = 1U; i < n; ++i) { + const auto v = result.GetElement(i).template Get<double>(); + min = std::min(min, v, comp); + max = std::max(max, v, comp); + } + + UNIT_ASSERT_VALUES_EQUAL(*mm.first, min); + UNIT_ASSERT_VALUES_EQUAL(*mm.second, max); + } + + Y_UNIT_TEST_LLVM(TestTopByNthElement) { + const std::array<double, 10U> xxx = {{0.0, 13.0, -3.140, -7898.8, 210000.0, 17E13, 1212.0, -HUGE_VAL, 3673.0, -32764.0}}; + + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + std::array<TRuntimeNode, 10U> data; + std::transform(xxx.cbegin(), xxx.cend(), data.begin(), [&pb](double f) { return pb.NewDataLiteral(f); } ); + + const auto type = pb.NewDataType(NUdf::TDataType<double>::Id); + const auto list = pb.NewList(type, data); + + const auto comparator = [&](TRuntimeNode l, TRuntimeNode r) { return pb.AggrLess(pb.Abs(l), pb.Abs(r)); }; + + const auto n = 5ULL; + + const auto limit = pb.NewDataLiteral<ui64>(n); + const auto reserve = pb.ShiftLeft(limit, pb.NewDataLiteral<ui8>(1U)); + const auto last = pb.Decrement(limit); + + const auto pgmReturn = pb.Take(pb.NthElement(pb.Fold(list, pb.NewEmptyList(type), + [&](TRuntimeNode item, TRuntimeNode state) { + const auto size = pb.Length(state); + + return pb.If(pb.AggrLess(size, limit), + pb.If(pb.AggrLess(size, last), + pb.Append(state, item), pb.MakeHeap(pb.Append(state, item), comparator)), + pb.If(comparator(item, pb.Unwrap(pb.ToOptional(state), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0)), + pb.If(pb.AggrLess(size, reserve), + pb.Append(state, item), + pb.Take(pb.NthElement(pb.Prepend(item, pb.Skip(state, pb.NewDataLiteral<ui64>(1U))), last, comparator), limit) + ), + state + ) + ); + } + ), last, comparator), limit); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto& result = graph->GetValue(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), n); + + auto copy = xxx; + + const auto comp = [](double l, double r){ return std::abs(l) < std::abs(r); }; + NYql::FastNthElement(copy.begin(), copy.begin() + n - 1U, copy.end(), comp); + const auto mm = std::minmax_element(copy.begin(), copy.begin() + n, comp); + + double min = result.GetElement(0).template Get<double>(), max = min; + for (auto i = 1U; i < n; ++i) { + const auto v = result.GetElement(i).template Get<double>(); + min = std::min(min, v, comp); + max = std::max(max, v, comp); + } + + UNIT_ASSERT_VALUES_EQUAL(*mm.first, min); + UNIT_ASSERT_VALUES_EQUAL(*mm.second, max); + } +} +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_dict_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_dict_ut.cpp new file mode 100644 index 00000000000..893722216d9 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_dict_ut.cpp @@ -0,0 +1,431 @@ +#include "mkql_computation_node_ut.h" + +namespace NKikimr { +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLJoinDictNodeTest) { + Y_UNIT_TEST_LLVM(TestInner) { + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto structType = pb.NewStructType({ + {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, + {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)} + }); + + const auto list1 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3) + }); + const auto dict1 = pb.ToSortedDict(list1, true, + [&](TRuntimeNode item) { + return pb.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pb.Member(item, "Payload"); + }); + + const auto list2 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6) + }); + const auto dict2 = pb.ToSortedDict(list2, true, + [&](TRuntimeNode item) { + return pb.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pb.Member(item, "Payload"); + }); + + const auto pgmReturn = pb.JoinDict(dict1, true, dict2, true, EJoinKind::Inner); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestLeft) { + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto structType = pb.NewStructType({ + {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, + {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)} + }); + + const auto list1 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3) + }); + const auto dict1 = pb.ToSortedDict(list1, true, + [&](TRuntimeNode item) { + return pb.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pb.Member(item, "Payload"); + }); + + const auto list2 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6) + }); + const auto dict2 = pb.ToSortedDict(list2, true, + [&](TRuntimeNode item) { + return pb.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pb.Member(item, "Payload"); + }); + + const auto pgmReturn = pb.JoinDict(dict1, true, dict2, true, EJoinKind::Left); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A"); + UNIT_ASSERT(!tuple.GetElement(1)); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNIT_ASSERT(tuple.GetElement(1)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNIT_ASSERT(tuple.GetElement(1)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNIT_ASSERT(tuple.GetElement(1)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNIT_ASSERT(tuple.GetElement(1)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestRight) { + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto structType = pb.NewStructType({ + {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, + {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)} + }); + + const auto list1 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3) + }); + const auto dict1 = pb.ToSortedDict(list1, true, + [&](TRuntimeNode item) { + return pb.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pb.Member(item, "Payload"); + }); + + const auto list2 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6) + }); + const auto dict2 = pb.ToSortedDict(list2, true, + [&](TRuntimeNode item) { + return pb.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pb.Member(item, "Payload"); + }); + + const auto pgmReturn = pb.JoinDict(dict1, true, dict2, true, EJoinKind::Right); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNIT_ASSERT(tuple.GetElement(0)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNIT_ASSERT(tuple.GetElement(0)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(iterator.Next(tuple)); + UNIT_ASSERT(tuple.GetElement(0)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNIT_ASSERT(tuple.GetElement(0)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(iterator.Next(tuple)); + UNIT_ASSERT(!tuple.GetElement(0)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Z"); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestFull) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto structType = pb.NewStructType({ + {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, + {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)} + }); + + const auto list1 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3) + }); + const auto dict1 = pb.ToSortedDict(list1, true, + [&](TRuntimeNode item) { + return pb.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pb.Member(item, "Payload"); + }); + + const auto list2 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6) + }); + const auto dict2 = pb.ToSortedDict(list2, true, + [&](TRuntimeNode item) { + return pb.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pb.Member(item, "Payload"); + }); + + const auto pgmReturn = pb.JoinDict(dict1, true, dict2, true, EJoinKind::Full); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNIT_ASSERT(tuple.GetElement(0)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A"); + UNIT_ASSERT(!tuple.GetElement(1)); + UNIT_ASSERT(iterator.Next(tuple)); + UNIT_ASSERT(tuple.GetElement(0)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNIT_ASSERT(tuple.GetElement(1)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNIT_ASSERT(tuple.GetElement(0)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNIT_ASSERT(tuple.GetElement(1)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(iterator.Next(tuple)); + UNIT_ASSERT(tuple.GetElement(0)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNIT_ASSERT(tuple.GetElement(1)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNIT_ASSERT(tuple.GetElement(0)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNIT_ASSERT(tuple.GetElement(1)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(iterator.Next(tuple)); + UNIT_ASSERT(!tuple.GetElement(0)); + UNIT_ASSERT(tuple.GetElement(1)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Z"); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + + Y_UNIT_TEST_LLVM(TestInnerFlat) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1U); + const auto key2 = pb.NewDataLiteral<ui32>(2U); + const auto key3 = pb.NewDataLiteral<ui32>(3U); + const auto key4 = pb.NewDataLiteral<ui32>(4U); + const auto key5 = pb.NewDataLiteral<ui32>(5U); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("D"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("E"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("F"); + const auto payload7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("G"); + const auto payload8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("H"); + + const auto structType = pb.NewStructType({ + {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, + {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)} + }); + + const auto list1 = pb.NewList(structType, { + pb.NewStruct(structType, {{"Key", key1}, {"Payload", payload1}}), + pb.NewStruct(structType, {{"Key", key2}, {"Payload", payload2}}), + pb.NewStruct(structType, {{"Key", key3}, {"Payload", payload3}}), + pb.NewStruct(structType, {{"Key", key4}, {"Payload", payload4}}) + }); + + const auto list2 = pb.NewList(structType, { + pb.NewStruct(structType, {{"Key", key2}, {"Payload", payload8}}), + pb.NewStruct(structType, {{"Key", key3}, {"Payload", payload7}}), + pb.NewStruct(structType, {{"Key", key4}, {"Payload", payload6}}), + pb.NewStruct(structType, {{"Key", key5}, {"Payload", payload5}}) + }); + + const auto listList = pb.NewList(pb.NewListType(structType), {list1, list2}); + + const auto pgmReturn = pb.FlatMap(listList, + [&](TRuntimeNode left) { + const auto dict1 = pb.ToSortedDict(left, false, + [&](TRuntimeNode item) { + return pb.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pb.Member(item, "Payload"); + }, false, 0); + return pb.FlatMap(listList, + [&](TRuntimeNode right) { + const auto dict2 = pb.ToSortedDict(right, false, + [&](TRuntimeNode item) { + return pb.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pb.Member(item, "Payload"); + }, false, 0); + return pb.JoinDict(dict1, false, dict2, false, EJoinKind::Inner); + }); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto list = graph->GetValue(); + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 14U); + + const auto iterator = list.GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "A"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "A"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "B"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "C"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "D"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "D"); + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "H"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "G"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "D"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "F"); + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "H"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "B"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "G"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "C"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "F"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "D"); + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "H"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "H"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "G"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "G"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "F"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "F"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "E"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "E"); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_ut.cpp new file mode 100644 index 00000000000..89e2827eb41 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_ut.cpp @@ -0,0 +1,329 @@ +#include "mkql_computation_node_ut.h" +#include <ydb/library/yql/minikql/mkql_runtime_version.h> + +namespace NKikimr { +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLCommonJoinCoreTupleTest) { + Y_UNIT_TEST_LLVM(Inner) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto indexType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto optionalType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({optionalType, optionalType, optionalType, indexType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-1)), pb.NewDataLiteral<ui32>(0)}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2)), pb.NewDataLiteral<ui32>(0)}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-3)), pb.NewDataLiteral<ui32>(1)}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(optionalType), pb.NewDataLiteral<ui32>(1)}); + + const auto list = pb.NewList(tupleType, {data1, data3, data2, data4}); + + const auto outputType = pb.NewFlowType(pb.NewMultiType({optionalType, optionalType})); + const auto pgmReturn = pb.Collect(pb.CommonJoinCore(pb.ToFlow(list), EJoinKind::Inner, {0U, 0U}, {1U, 1U}, {}, {2U}, 0ULL, std::nullopt, EAnyJoinSettings::None, 3U, outputType)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(InnerOrderLeftFirst) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto indexType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto optionalType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({optionalType, optionalType, optionalType, indexType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-1)), pb.NewDataLiteral<ui32>(0)}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2)), pb.NewDataLiteral<ui32>(0)}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-3)), pb.NewDataLiteral<ui32>(1)}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(optionalType), pb.NewDataLiteral<ui32>(1)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto outputType = pb.NewFlowType(pb.NewMultiType({optionalType, optionalType})); + const auto pgmReturn = pb.Collect(pb.CommonJoinCore(pb.ToFlow(list), EJoinKind::Inner, {0U, 0U}, {1U, 1U}, {}, {2U}, 0ULL, {0U}, EAnyJoinSettings::None, 3U, outputType)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(InnerOrderRightFirst) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto indexType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto optionalType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({optionalType, optionalType, optionalType, indexType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-1)), pb.NewDataLiteral<ui32>(0)}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2)), pb.NewDataLiteral<ui32>(0)}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-3)), pb.NewDataLiteral<ui32>(1)}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(optionalType), pb.NewDataLiteral<ui32>(1)}); + + const auto list = pb.NewList(tupleType, {data3, data4, data1, data2}); + + const auto outputType = pb.NewFlowType(pb.NewMultiType({optionalType, optionalType})); + const auto pgmReturn = pb.Collect(pb.CommonJoinCore(pb.ToFlow(list), EJoinKind::Inner, {0U, 0U}, {1U, 1U}, {}, {2U}, 0ULL, {1U}, EAnyJoinSettings::None, 3U, outputType)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } +} +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u +Y_UNIT_TEST_SUITE(TMiniKQLCommonJoinCoreWideTest) { + Y_UNIT_TEST_LLVM(Inner) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto indexType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto optionalType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({optionalType, optionalType, optionalType, indexType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-1)), pb.NewDataLiteral<ui32>(0)}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2)), pb.NewDataLiteral<ui32>(0)}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-3)), pb.NewDataLiteral<ui32>(1)}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(optionalType), pb.NewDataLiteral<ui32>(1)}); + + const auto list = pb.NewList(tupleType, {data1, data3, data2, data4}); + + const auto outputType = pb.NewFlowType(pb.NewMultiType({optionalType, optionalType})); + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.CommonJoinCore(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U), pb.Nth(item, 3U)}; }), + EJoinKind::Inner, {0U, 0U}, {1U, 1U}, {}, {2U}, 0ULL, std::nullopt, EAnyJoinSettings::None, 3U, outputType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(InnerOrderLeftFirst) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto indexType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto optionalType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({optionalType, optionalType, optionalType, indexType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-1)), pb.NewDataLiteral<ui32>(0)}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2)), pb.NewDataLiteral<ui32>(0)}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-3)), pb.NewDataLiteral<ui32>(1)}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(optionalType), pb.NewDataLiteral<ui32>(1)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto outputType = pb.NewFlowType(pb.NewMultiType({optionalType, optionalType})); + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.CommonJoinCore(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U), pb.Nth(item, 3U)}; }), + EJoinKind::Inner, {0U, 0U}, {1U, 1U}, {}, {2U}, 0ULL, {0U}, EAnyJoinSettings::None, 3U, outputType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(InnerOrderRightFirst) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto indexType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto optionalType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({optionalType, optionalType, optionalType, indexType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-1)), pb.NewDataLiteral<ui32>(0)}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2)), pb.NewDataLiteral<ui32>(0)}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-3)), pb.NewDataLiteral<ui32>(1)}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(optionalType), pb.NewDataLiteral<ui32>(1)}); + + const auto list = pb.NewList(tupleType, {data3, data4, data1, data2}); + + const auto outputType = pb.NewFlowType(pb.NewMultiType({optionalType, optionalType})); + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.CommonJoinCore(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U), pb.Nth(item, 3U)}; }), + EJoinKind::Inner, {0U, 0U}, {1U, 1U}, {}, {2U}, 0ULL, {1U}, EAnyJoinSettings::None, 3U, outputType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(ExclusionOrderLeftFirstAny) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto indexType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto stringType = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id); + const auto optStrType = pb.NewOptionalType(stringType); + const auto optionalType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({optionalType, optStrType, optStrType, indexType}); + + const auto value1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("very long value 1")); + const auto value2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("very long value 2")); + const auto value3 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("very long value 3")); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(optStrType), value1, pb.NewDataLiteral<ui32>(1)}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewEmptyOptional(optStrType), value2, pb.NewDataLiteral<ui32>(1)}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(optStrType), value3, pb.NewDataLiteral<ui32>(1)}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(optStrType), pb.NewEmptyOptional(optStrType), pb.NewDataLiteral<ui32>(1)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto outputType = pb.NewFlowType(pb.NewMultiType({optStrType, optStrType})); + + const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("ACHTUNG MINEN!"); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.CommonJoinCore(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.NewOptional(pb.Unwrap(pb.Nth(item, 2U), landmine, __FILE__, __LINE__, 0)), pb.Nth(item, 3U)}; }), + EJoinKind::Exclusion, {1U, 0U}, {2U, 1U}, {0U}, {0U}, 0ULL, {0U}, EAnyJoinSettings::Right, 3U, outputType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 1"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(ExclusionOrderRightFirstAny) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto indexType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto stringType = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id); + const auto optStrType = pb.NewOptionalType(stringType); + const auto optionalType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({optionalType, optStrType, optStrType, indexType}); + + const auto value1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("very long value 1")); + const auto value2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("very long value 2")); + const auto value3 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("very long value 3")); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), value1, pb.NewEmptyOptional(optStrType), pb.NewDataLiteral<ui32>(0)}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), value2, pb.NewEmptyOptional(optStrType), pb.NewDataLiteral<ui32>(0)}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), value3, pb.NewEmptyOptional(optStrType), pb.NewDataLiteral<ui32>(0)}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(optStrType), pb.NewEmptyOptional(optStrType), pb.NewDataLiteral<ui32>(0)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto outputType = pb.NewFlowType(pb.NewMultiType({optStrType, optStrType})); + + const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("ACHTUNG MINEN!"); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.CommonJoinCore(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.NewOptional(pb.Unwrap(pb.Nth(item, 1U), landmine, __FILE__, __LINE__, 0)), pb.Nth(item, 2U), pb.Nth(item, 3U)}; }), + EJoinKind::Exclusion, {1U, 0U}, {2U, 1U}, {0U}, {0U}, 0ULL, {1U}, EAnyJoinSettings::Left, 3U, outputType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(1)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long value 1"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } +} +#endif +} +} + diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_listfromrange_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_listfromrange_ut.cpp new file mode 100644 index 00000000000..1ba1c92bdb0 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_listfromrange_ut.cpp @@ -0,0 +1,469 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> + + +namespace NKikimr { +namespace NMiniKQL { + +namespace { +template<bool UseLLVM, typename T> +TRuntimeNode MakeList(TSetup<UseLLVM>& setup, T Start, T End, i64 Step, const auto dateType) { + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto start = pb.Unwrap(pb.ToIntegral(pb.NewDataLiteral<T>(Start), dateType), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0); + + const auto end = pb.Unwrap(pb.ToIntegral(pb.NewDataLiteral<T>(End), dateType), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0); + + const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>( + NUdf::TStringRef((const char*)&Step, sizeof(Step))); + + return pb.Collect(pb.ToFlow(pb.ListFromRange(start, end, step))); +} +} + + +Y_UNIT_TEST_SUITE(TMiniKQLListFromRangeTest) { + Y_UNIT_TEST_LLVM(TestCorrectDate) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + ui16 start = 140; + ui16 end = 150; + i64 step = 86400000000LL; + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true); + + const auto dates = MakeList(setup, start, end, step, dateType); + const auto graph = setup.BuildGraph(dates); + const auto list = graph->GetValue(); + const auto iterator = list.GetListIterator(); + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), end - start); + NUdf::TUnboxedValue item; + for (size_t i = 0; i < end - start; i++) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), start + i); + } + UNIT_ASSERT(!iterator.Next(item)); + } + Y_UNIT_TEST_LLVM(TestCorrectDateReverse) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + ui16 start = 150; + ui16 end = 140; + i64 step = -86400000000LL; + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true); + + const auto dates = MakeList(setup, start, end, step, dateType); + const auto graph = setup.BuildGraph(dates); + const auto list = graph->GetValue(); + const auto iterator = list.GetListIterator(); + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), start - end); + NUdf::TUnboxedValue item; + for (int i = 0; i < start - end; i++) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), start - i); + } + UNIT_ASSERT(!iterator.Next(item)); + } + Y_UNIT_TEST_LLVM(TestCorrectDatetime) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + ui32 start = 140; + ui32 end = 150; + i64 step = 1000000LL; + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Datetime, true); + + const auto dates = MakeList(setup, start, end, step, dateType); + const auto graph = setup.BuildGraph(dates); + const auto list = graph->GetValue(); + const auto iterator = list.GetListIterator(); + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), end - start); + NUdf::TUnboxedValue item; + for (size_t i = 0; i < end - start; i++) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), start + i); + } + UNIT_ASSERT(!iterator.Next(item)); + } + Y_UNIT_TEST_LLVM(TestCorrectTimestamp) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + ui64 start = 140; + ui64 end = 150; + i64 step = 1LL; + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Timestamp, true); + + const auto dates = MakeList(setup, start, end, step, dateType); + const auto graph = setup.BuildGraph(dates); + const auto list = graph->GetValue(); + const auto iterator = list.GetListIterator(); + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), end - start); + NUdf::TUnboxedValue item; + for (size_t i = 0; i < end - start; i++) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), start + i); + } + UNIT_ASSERT(!iterator.Next(item)); + } + Y_UNIT_TEST_LLVM(TestWrongIntervalForDate) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + ui16 start = 140; + ui16 end = 150; + i64 step = 86400000001LL; + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true); + + const auto dates = MakeList(setup, start, end, step, dateType); + const auto graph = setup.BuildGraph(dates); + const auto list = graph->GetValue(); + const auto iterator = list.GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 0); + UNIT_ASSERT(!iterator.Next(item)); + } + Y_UNIT_TEST_LLVM(TestWrongIntervalForDatetime) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + ui32 start = 140; + ui32 end = 150; + i64 step = 1000003LL; + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Datetime, true); + + const auto dates = MakeList(setup, start, end, step, dateType); + const auto graph = setup.BuildGraph(dates); + const auto list = graph->GetValue(); + const auto iterator = list.GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 0); + UNIT_ASSERT(!iterator.Next(item)); + } + Y_UNIT_TEST_LLVM(TestWrongStartType) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto value0 = ui32(1000000); + const auto start = pb.NewDataLiteral<ui32>(value0); + + const auto value1 = ui32(1000005); + const auto end = pb.NewDataLiteral<NUdf::EDataSlot::Datetime>( + NUdf::TStringRef((const char*)&value1, sizeof(value1))); + + const auto value2 = i64(1000001LL); + const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>( + NUdf::TStringRef((const char*)&value2, sizeof(value2))); + + UNIT_ASSERT_EXCEPTION(pb.ListFromRange(start, end, step), yexception); + } + Y_UNIT_TEST_LLVM(TestWrongEndType) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto value0 = ui32(1000000); + const auto start = pb.NewDataLiteral<NUdf::EDataSlot::Datetime>( + NUdf::TStringRef((const char*)&value0, sizeof(value0))); + + const auto value1 = ui32(1000005); + const auto end = pb.NewDataLiteral<ui32>(value1); + + const auto value2 = i64(1000001LL); + const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>( + NUdf::TStringRef((const char*)&value2, sizeof(value2))); + + UNIT_ASSERT_EXCEPTION(pb.ListFromRange(start, end, step), yexception); + } + Y_UNIT_TEST_LLVM(TestWrongStepType) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto value0 = ui32(1000000); + const auto start = pb.NewDataLiteral<NUdf::EDataSlot::Datetime>( + NUdf::TStringRef((const char*)&value0, sizeof(value0))); + + const auto value1 = ui32(1000005); + const auto end = pb.NewDataLiteral<NUdf::EDataSlot::Datetime>( + NUdf::TStringRef((const char*)&value1, sizeof(value1))); + + const auto value2 = i64(1000001LL); + const auto step = pb.NewDataLiteral<ui32>(value2); + + UNIT_ASSERT_EXCEPTION(pb.ListFromRange(start, end, step), yexception); + } + Y_UNIT_TEST_LLVM(TestEmptyListDate) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + ui16 start = 150; + ui16 end = 144; + i64 step = 86400000000LL; + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true); + + const auto dates = MakeList(setup, start, end, step, dateType); + const auto graph = setup.BuildGraph(dates); + const auto list = graph->GetValue(); + const auto iterator = list.GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 0); + UNIT_ASSERT(!iterator.Next(item)); + } + Y_UNIT_TEST_LLVM(TestWrongStartEndTypes) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto value0 = ui16(140); + const auto start = pb.NewDataLiteral<NUdf::EDataSlot::Date>( + NUdf::TStringRef((const char*)&value0, sizeof(value0))); + + const auto value1 = ui32(140 * 60 * 60 * 24 + 5); + const auto end = pb.NewDataLiteral<NUdf::EDataSlot::Datetime>( + NUdf::TStringRef((const char*)&value1, sizeof(value1))); + + const auto value2 = i64(2000000LL); // 2 Seconds + const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>( + NUdf::TStringRef((const char*)&value2, sizeof(value2))); + + UNIT_ASSERT_EXCEPTION(pb.ListFromRange(start, end, step), yexception); + } + Y_UNIT_TEST_LLVM(TestMinOverflowForDate) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + ui16 start = 4; + ui16 end = 0; + i64 step = -518400000000LL; // -6 days + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true); + + const auto dates = MakeList(setup, start, end, step, dateType); + const auto graph = setup.BuildGraph(dates); + const auto list = graph->GetValue(); + const auto iterator = list.GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), start); + UNIT_ASSERT(!iterator.Next(item)); + } + Y_UNIT_TEST_LLVM(TestMinOverflowForDatetime) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + ui32 start = 9; + ui32 end = 0; + i64 step = -10000000LL; // -10 seconds + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Datetime, true); + + const auto dates = MakeList(setup, start, end, step, dateType); + const auto graph = setup.BuildGraph(dates); + const auto list = graph->GetValue(); + const auto iterator = list.GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), start); + UNIT_ASSERT(!iterator.Next(item)); + } + Y_UNIT_TEST_LLVM(TestMinOverflowForTimestamp) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + ui64 start = 100; + ui64 end = 10; + i64 step = -110LL; // -110 microseconds + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Timestamp, true); + + const auto dates = MakeList(setup, start, end, step, dateType); + const auto graph = setup.BuildGraph(dates); + const auto list = graph->GetValue(); + const auto iterator = list.GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), start); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestMaxOverflowForDate) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + ui16 start = 100; + ui16 end = NYql::NUdf::MAX_DATE - 1; + i64 step = (NYql::NUdf::MAX_DATE - 1) * 24LL * 60 * 60 * 1000000; + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true); + + const auto dates = MakeList(setup, start, end, step, dateType); + const auto graph = setup.BuildGraph(dates); + const auto list = graph->GetValue(); + const auto iterator = list.GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), start); + UNIT_ASSERT(!iterator.Next(item)); + } + Y_UNIT_TEST_LLVM(TestMaxOverflowForDatetime) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + ui32 start = NYql::NUdf::MAX_DATETIME - 123; + ui32 end = NYql::NUdf::MAX_DATETIME - 1; + i64 step = (NYql::NUdf::MAX_DATETIME - 1) * 1000000LL; + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Datetime, true); + + const auto dates = MakeList(setup, start, end, step, dateType); + const auto graph = setup.BuildGraph(dates); + const auto list = graph->GetValue(); + const auto iterator = list.GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), start); + UNIT_ASSERT(!iterator.Next(item)); + } + Y_UNIT_TEST_LLVM(TestMaxOverflowForTimestamp) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + ui64 start = NYql::NUdf::MAX_TIMESTAMP - 123; + ui64 end = NYql::NUdf::MAX_TIMESTAMP - 1; + i64 step = NYql::NUdf::MAX_TIMESTAMP - 1; + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Timestamp, true); + + const auto dates = MakeList(setup, start, end, step, dateType); + const auto graph = setup.BuildGraph(dates); + const auto list = graph->GetValue(); + const auto iterator = list.GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), start); + UNIT_ASSERT(!iterator.Next(item)); + } + Y_UNIT_TEST_LLVM(TestDifferentTimezonesForTzDate) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true); + const auto canada = pb.NewDataLiteral<ui16>(375U); + const auto europe = pb.NewDataLiteral<ui16>(459U); + const auto value2 = i64(24LL * 60 * 60 * 1000000); // 1 Day + const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>( + NUdf::TStringRef((const char*)&value2, sizeof(value2))); + + const auto day1 = pb.ToIntegral(pb.NewDataLiteral<ui16>(123), dateType); + const auto day2 = pb.ToIntegral(pb.NewDataLiteral<ui16>(123 + 5), dateType); + const auto date1 = pb.Unwrap(pb.AddTimezone(day1, canada), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0); + const auto date2 = pb.Unwrap(pb.AddTimezone(day2, europe), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0); + const auto dates = pb.ListFromRange(date1, date2, step); + + const auto graph = setup.BuildGraph(dates); + const auto list = graph->GetValue(); + const auto iterator = list.GetListIterator(); + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 5); + NUdf::TUnboxedValue item; + for (size_t i = 123; i < 123 + 5; i++) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), i); + UNIT_ASSERT_VALUES_EQUAL(item.GetTimezoneId(), 375U); + } + } + Y_UNIT_TEST_LLVM(TestSameTimezonesForTzDate) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true); + const auto canada = pb.NewDataLiteral<ui16>(375U); + const auto value2 = i64(24LL * 60 * 60 * 1000000); // 1 Day + const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>( + NUdf::TStringRef((const char*)&value2, sizeof(value2))); + + const auto day1 = pb.ToIntegral(pb.NewDataLiteral<ui16>(123), dateType); + const auto day2 = pb.ToIntegral(pb.NewDataLiteral<ui16>(123 + 5), dateType); + const auto date1 = pb.Unwrap(pb.AddTimezone(day1, canada), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0); + const auto date2 = pb.Unwrap(pb.AddTimezone(day2, canada), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0); + const auto dates = pb.ListFromRange(date1, date2, step); + + const auto graph = setup.BuildGraph(dates); + const auto list = graph->GetValue(); + const auto iterator = list.GetListIterator(); + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 5); + NUdf::TUnboxedValue item; + for (size_t i = 123; i < 123 + 5; i++) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), i); + UNIT_ASSERT_VALUES_EQUAL(item.GetTimezoneId(), 375U); + } + } + Y_UNIT_TEST_LLVM(TestDifferentTimezonesForTzDatetime) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Datetime, true); + const auto canada = pb.NewDataLiteral<ui16>(375U); + const auto europe = pb.NewDataLiteral<ui16>(459U); + const auto value2 = i64(1000000LL); // 1 Second + const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>( + NUdf::TStringRef((const char*)&value2, sizeof(value2))); + + const auto day1 = pb.ToIntegral(pb.NewDataLiteral<ui32>(123), dateType); + const auto day2 = pb.ToIntegral(pb.NewDataLiteral<ui32>(123 + 5), dateType); + const auto date1 = pb.Unwrap(pb.AddTimezone(day1, canada), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0); + const auto date2 = pb.Unwrap(pb.AddTimezone(day2, europe), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0); + const auto dates = pb.ListFromRange(date1, date2, step); + + const auto graph = setup.BuildGraph(dates); + const auto list = graph->GetValue(); + const auto iterator = list.GetListIterator(); + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 5); + NUdf::TUnboxedValue item; + for (size_t i = 123; i < 123 + 5; i++) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), i); + UNIT_ASSERT_VALUES_EQUAL(item.GetTimezoneId(), 375U); + } + } + Y_UNIT_TEST_LLVM(TestDifferentTimezonesForTzTimestamp) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dateType = pb.NewDataType(NUdf::EDataSlot::Timestamp, true); + const auto europe = pb.NewDataLiteral<ui16>(459U); + const auto canada = pb.NewDataLiteral<ui16>(375U); + const auto value2 = i64(1LL); // 1 Microsecond + const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>( + NUdf::TStringRef((const char*)&value2, sizeof(value2))); + + const auto day1 = pb.ToIntegral(pb.NewDataLiteral<ui64>(123), dateType); + const auto day2 = pb.ToIntegral(pb.NewDataLiteral<ui64>(123 + 5), dateType); + const auto date1 = pb.Unwrap(pb.AddTimezone(day1, europe), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0); + const auto date2 = pb.Unwrap(pb.AddTimezone(day2, canada), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0); + const auto dates = pb.ListFromRange(date1, date2, step); + + const auto graph = setup.BuildGraph(dates); + const auto list = graph->GetValue(); + const auto iterator = list.GetListIterator(); + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 5); + NUdf::TUnboxedValue item; + for (size_t i = 123; i < 123 + 5; i++) { + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), i); + UNIT_ASSERT_VALUES_EQUAL(item.GetTimezoneId(), 459U); + } + } + + Y_UNIT_TEST_LLVM(TestResverseUnsignedShorts) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto from = pb.NewDataLiteral<ui16>(60000U); + const auto to = pb.NewDataLiteral<ui16>(59990U); + const auto step = pb.NewDataLiteral<i16>(-2); + + const auto dates = pb.Collect(pb.ToFlow(pb.ListFromRange(from, to, step))); + + const auto graph = setup.BuildGraph(dates); + const auto list = graph->GetValue(); + UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 5UL); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(0U).template Get<ui16>(), 60000U); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(1U).template Get<ui16>(), 59998U); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(2U).template Get<ui16>(), 59996U); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(3U).template Get<ui16>(), 59994U); + UNIT_ASSERT_VALUES_EQUAL(list.GetElement(4U).template Get<ui16>(), 59992U); + } +} +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_map_join_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_map_join_ut.cpp new file mode 100644 index 00000000000..45f039e2099 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_map_join_ut.cpp @@ -0,0 +1,1150 @@ +#include "mkql_computation_node_ut.h" +#include <ydb/library/yql/minikql/mkql_runtime_version.h> + +namespace NKikimr { +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLMapJoinCoreTest) { + Y_UNIT_TEST_LLVM(TestInnerOnTuple) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto optionalUi64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id, true); + const auto tupleType = pb.NewTupleType({optionalUi64Type, optionalUi64Type}); + const auto emptyOptionalUi64 = pb.NewEmptyOptional(optionalUi64Type); + + const auto key1 = pb.NewTuple(tupleType, { + pb.NewOptional(pb.NewDataLiteral<ui64>(1)), + pb.NewOptional(pb.NewDataLiteral<ui64>(1)), + }); + const auto key2 = pb.NewTuple(tupleType, { + pb.NewOptional(pb.NewDataLiteral<ui64>(2)), + pb.NewOptional(pb.NewDataLiteral<ui64>(2)), + }); + const auto key3 = pb.NewTuple(tupleType, { + pb.NewOptional(pb.NewDataLiteral<ui64>(3)), + emptyOptionalUi64, + }); + const auto key4 = pb.NewTuple(tupleType, { + pb.NewOptional(pb.NewDataLiteral<ui64>(4)), + pb.NewOptional(pb.NewDataLiteral<ui64>(4)), + }); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto structType = pb.NewStructType({ + {"Key", tupleType}, + {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)} + }); + + const auto list1 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3) + }); + + const auto list2 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6) + }); + const auto dict2 = pb.ToSortedDict(list2, false, + [&](TRuntimeNode item) { + return pb.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pb.AddMember(pb.NewEmptyStruct(), "Payload", pb.Member(item, "Payload")); + }); + + const auto resultType = pb.NewFlowType(pb.NewStructType({ + {"Left", pb.NewDataType(NUdf::TDataType<char*>::Id)}, + {"Right", pb.NewDataType(NUdf::TDataType<char*>::Id)}, + })); + + const auto pgmReturn = pb.Collect(pb.MapJoinCore(pb.ToFlow(list1), dict2, EJoinKind::Inner, {0U}, {1U, 0U}, {0U, 1U}, resultType)); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + + Y_UNIT_TEST_LLVM(TestInner) { + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(3); + const auto key4 = pb.NewDataLiteral<ui32>(4); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto structType = pb.NewStructType({ + {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, + {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)} + }); + + const auto list1 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3) + }); + + const auto list2 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6) + }); + const auto dict2 = pb.ToHashedDict(list2, false, + [&](TRuntimeNode item) { + return pb.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pb.AddMember(pb.NewEmptyStruct(), "Payload", pb.Member(item, "Payload")); + }); + + const auto resultType = pb.NewFlowType(pb.NewStructType({ + {"Left", pb.NewDataType(NUdf::TDataType<char*>::Id)}, + {"Right", pb.NewDataType(NUdf::TDataType<char*>::Id)}, + })); + + const auto pgmReturn = pb.Collect(pb.MapJoinCore(pb.ToFlow(list1), dict2, EJoinKind::Inner, {0U}, {1U, 0U}, {0U, 1U}, resultType)); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestInnerMulti) { + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto structType = pb.NewStructType({ + {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, + {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)} + }); + + const auto list1 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3) + }); + + const auto list2 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6) + }); + const auto dict2 = pb.ToHashedDict(list2, true, + [&](TRuntimeNode item) { + return pb.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pb.AddMember(pb.NewEmptyStruct(), "Payload", pb.Member(item, "Payload")); + }); + + const auto resultType = pb.NewFlowType(pb.NewStructType({ + {"Left", pb.NewDataType(NUdf::TDataType<char*>::Id)}, + {"Right", pb.NewDataType(NUdf::TDataType<char*>::Id)}, + })); + + const auto pgmReturn = pb.Collect(pb.MapJoinCore(pb.ToFlow(list1), dict2, EJoinKind::Inner, {0U}, {1U, 0U}, {0U, 1U}, resultType)); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestLeft) { + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(3); + const auto key4 = pb.NewDataLiteral<ui32>(4); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto structType = pb.NewStructType({ + {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, + {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)} + }); + const auto list1 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3) + }); + + const auto list2 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6) + }); + const auto dict2 = pb.ToHashedDict(list2, false, + [&](TRuntimeNode item) { + return pb.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pb.AddMember(pb.NewEmptyStruct(), "Payload", pb.Member(item, "Payload")); + }); + + const auto resultType = pb.NewFlowType(pb.NewStructType({ + {"Left", pb.NewDataType(NUdf::TDataType<char*>::Id)}, + {"Right", pb.NewDataType(NUdf::TDataType<char*>::Id)}, + })); + + const auto pgmReturn = pb.Collect(pb.MapJoinCore(pb.ToFlow(list1), dict2, EJoinKind::Left, {0U}, {1U, 0U}, {0U, 1U}, resultType)); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A"); + UNIT_ASSERT(!tuple.GetElement(1)); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestLeftMulti) { + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto structType = pb.NewStructType({ + {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, + {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)} + }); + const auto list1 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3) + }); + + const auto list2 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6) + }); + const auto dict2 = pb.ToHashedDict(list2, true, + [&](TRuntimeNode item) { + return pb.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pb.AddMember(pb.NewEmptyStruct(), "Payload", pb.Member(item, "Payload")); + }); + + const auto resultType = pb.NewFlowType(pb.NewStructType({ + {"Left", pb.NewDataType(NUdf::TDataType<char*>::Id)}, + {"Right", pb.NewDataType(NUdf::TDataType<char*>::Id)}, + })); + + const auto pgmReturn = pb.Collect(pb.MapJoinCore(pb.ToFlow(list1), dict2, EJoinKind::Left, {0U}, {1U, 0U}, {0U, 1U}, resultType)); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A"); + UNIT_ASSERT(!tuple.GetElement(1)); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestLeftSemi) { + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto structType = pb.NewStructType({ + {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, + {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)} + }); + const auto list1 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3) + }); + + const auto list2 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6) + }); + const auto dict2 = pb.ToHashedDict(list2, true, + [&](TRuntimeNode item) { + return pb.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pb.AddMember(pb.NewEmptyStruct(), "Payload", pb.Member(item, "Payload")); + }); + + const auto resultType = pb.NewFlowType(pb.NewStructType({ + {"Left", pb.NewDataType(NUdf::TDataType<char*>::Id)}, + {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, + })); + + const auto pgmReturn = pb.Collect(pb.MapJoinCore(pb.ToFlow(list1), dict2, EJoinKind::LeftSemi, {0U}, {1U, 1U, 0U, 0U}, {}, resultType)); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(0).Get<ui32>(), 2); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "B"); + UNIT_ASSERT(iterator.Next(tuple)); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(0).Get<ui32>(), 2); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "C"); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestLeftOnly) { + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto structType = pb.NewStructType({ + {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, + {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)} + }); + const auto list1 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3) + }); + + const auto list2 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6) + }); + const auto dict2 = pb.ToHashedDict(list2, true, + [&](TRuntimeNode item) { + return pb.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pb.AddMember(pb.NewEmptyStruct(), "Payload", pb.Member(item, "Payload")); + }); + + const auto resultType = pb.NewFlowType(pb.NewStructType({ + {"Left", pb.NewDataType(NUdf::TDataType<char*>::Id)}, + {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, + })); + + const auto pgmReturn = pb.Collect(pb.MapJoinCore(pb.ToFlow(list1), dict2, EJoinKind::LeftOnly, {0U}, {1U, 1U, 0U, 0U}, {}, resultType)); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(0).Get<ui32>(), 1); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "A"); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestLeftSemiWithNullKey) { + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key0 = pb.NewEmptyOptional(pb.NewDataType(NUdf::TDataType<ui32>::Id, true)); + const auto key1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1)); + const auto key2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2)); + const auto key3 = pb.NewOptional(pb.NewDataLiteral<ui32>(2)); + const auto key4 = pb.NewOptional(pb.NewDataLiteral<ui32>(3)); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto structType = pb.NewStructType({ + {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id, true)}, + {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)} + }); + const auto list1 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key0), "Payload", payload4), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3) + }); + + const auto list2 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key0), "Payload", payload3), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6) + }); + + const auto dict2 = pb.ToHashedDict(list2, true, + [&](TRuntimeNode item) { + return pb.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pb.AddMember(pb.NewEmptyStruct(), "Payload", pb.Member(item, "Payload")); + }); + + const auto resultType = pb.NewFlowType(pb.NewStructType({ + {"Left", pb.NewDataType(NUdf::TDataType<char*>::Id)}, + {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, + })); + + const auto pgmReturn = pb.Collect(pb.MapJoinCore(pb.ToFlow(list1), dict2, EJoinKind::LeftSemi, {0U}, {1U, 1U, 0U, 0U}, {}, resultType)); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(0).Get<ui32>(), 2); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "B"); + UNIT_ASSERT(iterator.Next(tuple)); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(0).Get<ui32>(), 2); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "C"); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestLeftOnlyWithNullKey) { + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key0 = pb.NewEmptyOptional(pb.NewDataType(NUdf::TDataType<ui32>::Id, true)); + const auto key1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1)); + const auto key2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2)); + const auto key3 = pb.NewOptional(pb.NewDataLiteral<ui32>(2)); + const auto key4 = pb.NewOptional(pb.NewDataLiteral<ui32>(3)); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto structType = pb.NewStructType({ + {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id, true)}, + {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)} + }); + const auto list1 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key0), "Payload", payload4), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3) + }); + + const auto list2 = pb.NewList(structType, { + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key0), "Payload", payload3), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5), + pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6) + }); + + const auto dict2 = pb.ToHashedDict(list2, true, + [&](TRuntimeNode item) { + return pb.Member(item, "Key"); + }, + [&](TRuntimeNode item) { + return pb.AddMember(pb.NewEmptyStruct(), "Payload", pb.Member(item, "Payload")); + }); + + const auto resultType = pb.NewFlowType(pb.NewStructType({ + {"Left", pb.NewDataType(NUdf::TDataType<char*>::Id)}, + {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, + })); + + const auto pgmReturn = pb.Collect(pb.MapJoinCore(pb.ToFlow(list1), dict2, EJoinKind::LeftOnly, {0U}, {1U, 1U, 0U, 0U}, {}, resultType)); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNIT_ASSERT(!tuple.GetElement(0)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(0).Get<ui32>(), 1); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "A"); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } +} +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u +Y_UNIT_TEST_SUITE(TMiniKQLWideMapJoinCoreTest) { + Y_UNIT_TEST_LLVM(TestInner) { + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(4); + const auto key4 = pb.NewDataLiteral<ui32>(4); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + const auto dict2 = pb.ToHashedDict(list2, false, + [&](TRuntimeNode item) { + return pb.Nth(item, 0U); + }, + [&](TRuntimeNode item) { + return pb.NewTuple({pb.Nth(item, 1U)}); + }); + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + })); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.MapJoinCore(pb.ExpandMap(pb.ToFlow(list1), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + dict2, EJoinKind::Inner, {0U}, {1U, 0U}, {0U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestInnerMulti) { + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + const auto dict2 = pb.ToHashedDict(list2, true, + [&](TRuntimeNode item) { + return pb.Nth(item, 0U); + }, + [&](TRuntimeNode item) { + return pb.NewTuple({pb.Nth(item, 1U)}); + }); + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + })); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.MapJoinCore(pb.ExpandMap(pb.ToFlow(list1), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + dict2, EJoinKind::Inner, {0U}, {1U, 0U}, {0U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestLeft) { + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(3); + const auto key4 = pb.NewDataLiteral<ui32>(4); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + const auto dict2 = pb.ToHashedDict(list2, false, + [&](TRuntimeNode item) { + return pb.Nth(item, 0U); + }, + [&](TRuntimeNode item) { + return pb.NewTuple({pb.Nth(item, 1U)}); + }); + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + })); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.MapJoinCore(pb.ExpandMap(pb.ToFlow(list1), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + dict2, EJoinKind::Left, {0U}, {1U, 0U}, {0U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A"); + UNIT_ASSERT(!tuple.GetElement(1)); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestLeftMulti) { + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + const auto dict2 = pb.ToHashedDict(list2, true, + [&](TRuntimeNode item) { + return pb.Nth(item, 0U); + }, + [&](TRuntimeNode item) { + return pb.NewTuple({pb.Nth(item, 1U)}); + }); + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + })); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.MapJoinCore(pb.ExpandMap(pb.ToFlow(list1), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + dict2, EJoinKind::Left, {0U}, {1U, 0U}, {0U, 1U}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A"); + UNIT_ASSERT(!tuple.GetElement(1)); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X"); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y"); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestLeftSemi) { + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + const auto dict2 = pb.ToHashedDict(list2, true, + [&](TRuntimeNode item) { + return pb.Nth(item, 0U); + }, + [&](TRuntimeNode item) { + return pb.NewTuple({pb.Nth(item, 1U)}); + }); + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<ui32>::Id) + })); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.MapJoinCore(pb.ExpandMap(pb.ToFlow(list1), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + dict2, EJoinKind::LeftSemi, {0U}, {1U, 0U, 0U, 1U}, {}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestLeftOnly) { + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key1 = pb.NewDataLiteral<ui32>(1); + const auto key2 = pb.NewDataLiteral<ui32>(2); + const auto key3 = pb.NewDataLiteral<ui32>(2); + const auto key4 = pb.NewDataLiteral<ui32>(3); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + const auto dict2 = pb.ToHashedDict(list2, true, + [&](TRuntimeNode item) { + return pb.Nth(item, 0U); + }, + [&](TRuntimeNode item) { + return pb.NewTuple({pb.Nth(item, 1U)}); + }); + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<ui32>::Id) + })); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.MapJoinCore(pb.ExpandMap(pb.ToFlow(list1), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + dict2, EJoinKind::LeftOnly, {0U}, {1U, 0U, 0U, 1U}, {}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A"); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 1); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestLeftSemiWithNullKey) { + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key0 = pb.NewEmptyOptional(pb.NewDataType(NUdf::TDataType<ui32>::Id, true)); + const auto key1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1)); + const auto key2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2)); + const auto key3 = pb.NewOptional(pb.NewDataLiteral<ui32>(2)); + const auto key4 = pb.NewOptional(pb.NewDataLiteral<ui32>(3)); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id, true), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key0, payload4}), + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key0, payload3}), + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + const auto dict2 = pb.ToHashedDict(list2, true, + [&](TRuntimeNode item) { + return pb.Nth(item, 0U); + }, + [&](TRuntimeNode item) { + return pb.NewTuple({pb.Nth(item, 1U)}); + }); + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<ui32>::Id) + })); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.MapJoinCore(pb.ExpandMap(pb.ToFlow(list1), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + dict2, EJoinKind::LeftSemi, {0U}, {1U, 0U, 0U, 1U}, {}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B"); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C"); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } + + Y_UNIT_TEST_LLVM(TestLeftOnlyWithNullKey) { + for (ui32 pass = 0; pass < 1; ++pass) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto key0 = pb.NewEmptyOptional(pb.NewDataType(NUdf::TDataType<ui32>::Id, true)); + const auto key1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1)); + const auto key2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2)); + const auto key3 = pb.NewOptional(pb.NewDataLiteral<ui32>(2)); + const auto key4 = pb.NewOptional(pb.NewDataLiteral<ui32>(3)); + const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A"); + const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B"); + const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C"); + const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X"); + const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y"); + const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z"); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id, true), + pb.NewDataType(NUdf::TDataType<char*>::Id) + }); + + const auto list1 = pb.NewList(tupleType, { + pb.NewTuple({key0, payload4}), + pb.NewTuple({key1, payload1}), + pb.NewTuple({key2, payload2}), + pb.NewTuple({key3, payload3}) + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({key0, payload3}), + pb.NewTuple({key2, payload4}), + pb.NewTuple({key3, payload5}), + pb.NewTuple({key4, payload6}) + }); + + const auto dict2 = pb.ToHashedDict(list2, true, + [&](TRuntimeNode item) { + return pb.Nth(item, 0U); + }, + [&](TRuntimeNode item) { + return pb.NewTuple({pb.Nth(item, 1U)}); + }); + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<ui32>::Id) + })); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.MapJoinCore(pb.ExpandMap(pb.ToFlow(list1), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + dict2, EJoinKind::LeftOnly, {0U}, {1U, 0U, 0U, 1U}, {}, resultType), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue tuple; + + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "X"); + UNIT_ASSERT(!tuple.GetElement(1)); + UNIT_ASSERT(iterator.Next(tuple)); + UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A"); + UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 1); + UNIT_ASSERT(!iterator.Next(tuple)); + UNIT_ASSERT(!iterator.Next(tuple)); + } + } +} +#endif +} +} + diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_mapnext_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_mapnext_ut.cpp new file mode 100644 index 00000000000..a416803e447 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_mapnext_ut.cpp @@ -0,0 +1,172 @@ +#include "mkql_computation_node_ut.h" +#include <ydb/library/yql/minikql/mkql_runtime_version.h> + +namespace NKikimr { +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLMapNextTest) { + Y_UNIT_TEST_LLVM(OverStream) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui16>(10); + const auto data2 = pb.NewDataLiteral<ui16>(20); + const auto data3 = pb.NewDataLiteral<ui16>(30); + + const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id); + const auto optDataType = pb.NewOptionalType(dataType); + const auto tupleType = pb.NewTupleType({dataType, optDataType}); + + const auto list = pb.NewList(dataType, {data1, data2, data3}); + const auto pgmReturn = pb.MapNext(pb.Iterator(list, {}), + [&](TRuntimeNode item, TRuntimeNode nextItem) { + return pb.NewTuple(tupleType, {item, nextItem}); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui16>(), 10); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui16>(), 20); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui16>(), 20); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui16>(), 30); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui16>(), 30); + UNIT_ASSERT(!item.GetElement(1).HasValue()); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(OverSingleElementStream) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui16>(10); + + const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id); + const auto optDataType = pb.NewOptionalType(dataType); + const auto tupleType = pb.NewTupleType({dataType, optDataType}); + + const auto list = pb.NewList(dataType, {data1}); + const auto pgmReturn = pb.MapNext(pb.Iterator(list, {}), + [&](TRuntimeNode item, TRuntimeNode nextItem) { + return pb.NewTuple(tupleType, {item, nextItem}); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui16>(), 10); + UNIT_ASSERT(!item.GetElement(1).HasValue()); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(OverEmptyStream) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id); + const auto optDataType = pb.NewOptionalType(dataType); + const auto tupleType = pb.NewTupleType({dataType, optDataType}); + + const auto list = pb.NewList(dataType, {}); + const auto pgmReturn = pb.MapNext(pb.Iterator(list, {}), + [&](TRuntimeNode item, TRuntimeNode nextItem) { + return pb.NewTuple(tupleType, {item, nextItem}); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(OverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui16>(10); + const auto data2 = pb.NewDataLiteral<ui16>(20); + const auto data3 = pb.NewDataLiteral<ui16>(30); + + const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id); + const auto optDataType = pb.NewOptionalType(dataType); + const auto tupleType = pb.NewTupleType({dataType, optDataType}); + + const auto list = pb.NewList(dataType, {data1, data2, data3}); + const auto pgmReturn = pb.FromFlow(pb.MapNext(pb.ToFlow(pb.Iterator(list, {})), + [&](TRuntimeNode item, TRuntimeNode nextItem) { + return pb.NewTuple(tupleType, {item, nextItem}); + })); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui16>(), 10); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui16>(), 20); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui16>(), 20); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui16>(), 30); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui16>(), 30); + UNIT_ASSERT(!item.GetElement(1).HasValue()); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(OverSingleElementFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui16>(10); + + const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id); + const auto optDataType = pb.NewOptionalType(dataType); + const auto tupleType = pb.NewTupleType({dataType, optDataType}); + + const auto list = pb.NewList(dataType, {data1}); + const auto pgmReturn = pb.FromFlow(pb.MapNext(pb.ToFlow(pb.Iterator(list, {})), + [&](TRuntimeNode item, TRuntimeNode nextItem) { + return pb.NewTuple(tupleType, {item, nextItem}); + })); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui16>(), 10); + UNIT_ASSERT(!item.GetElement(1).HasValue()); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(OverEmptyFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id); + const auto optDataType = pb.NewOptionalType(dataType); + const auto tupleType = pb.NewTupleType({dataType, optDataType}); + + const auto list = pb.NewList(dataType, {}); + const auto pgmReturn = pb.FromFlow(pb.MapNext(pb.ToFlow(pb.Iterator(list, {})), + [&](TRuntimeNode item, TRuntimeNode nextItem) { + return pb.NewTuple(tupleType, {item, nextItem}); + })); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp new file mode 100644 index 00000000000..d7c4e8517ba --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp @@ -0,0 +1,305 @@ +#include "../mkql_multihopping.h" +#include <ydb/library/yql/minikql/mkql_node.h> +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_program_builder.h> +#include <ydb/library/yql/minikql/mkql_function_registry.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h> +#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> +#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h> + +#include <library/cpp/testing/unittest/registar.h> + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + TIntrusivePtr<IRandomProvider> CreateRandomProvider() { + return CreateDeterministicRandomProvider(1); + } + + TIntrusivePtr<ITimeProvider> CreateTimeProvider() { + return CreateDeterministicTimeProvider(10000000); + } + + TComputationNodeFactory GetAuxCallableFactory(TWatermark& watermark) { + return [&watermark](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* { + if (callable.GetType()->GetName() == "OneYieldStream") { + return new TExternalComputationNode(ctx.Mutables); + } else if (callable.GetType()->GetName() == "MultiHoppingCore") { + return WrapMultiHoppingCore(callable, ctx, watermark); + } + + return GetBuiltinFactory()(callable, ctx); + }; + } + + struct TSetup { + TSetup(TScopedAlloc& alloc) + : Alloc(alloc) + { + FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); + RandomProvider = CreateRandomProvider(); + TimeProvider = CreateTimeProvider(); + + Env.Reset(new TTypeEnvironment(Alloc)); + PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry)); + } + + THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) { + Explorer.Walk(pgm.GetNode(), *Env); + TComputationPatternOpts opts(Alloc.Ref(), *Env, GetAuxCallableFactory(Watermark), + FunctionRegistry.Get(), + NUdf::EValidateMode::None, NUdf::EValidatePolicy::Fail, "OFF", EGraphPerProcess::Multi); + Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts); + TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider); + return Pattern->Clone(compOpts); + } + + TIntrusivePtr<IFunctionRegistry> FunctionRegistry; + TIntrusivePtr<IRandomProvider> RandomProvider; + TIntrusivePtr<ITimeProvider> TimeProvider; + + TScopedAlloc& Alloc; + THolder<TTypeEnvironment> Env; + THolder<TProgramBuilder> PgmBuilder; + + TExploringNodeVisitor Explorer; + IComputationPattern::TPtr Pattern; + TWatermark Watermark; + }; + + struct TStreamWithYield : public NUdf::TBoxedValue { + TStreamWithYield(const TUnboxedValueVector& items, ui32 yieldPos, ui32 index) + : Items(items) + , YieldPos(yieldPos) + , Index(index) + {} + + private: + TUnboxedValueVector Items; + ui32 YieldPos; + ui32 Index; + + ui32 GetTraverseCount() const override { + return 0; + } + + NUdf::TUnboxedValue Save() const override { + return NUdf::TUnboxedValue::Zero(); + } + + void Load(const NUdf::TStringRef& state) override { + Y_UNUSED(state); + } + + NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) final { + if (Index >= Items.size()) { + return NUdf::EFetchStatus::Finish; + } + if (Index == YieldPos) { + return NUdf::EFetchStatus::Yield; + } + result = Items[Index++]; + return NUdf::EFetchStatus::Ok; + } + }; + + THolder<IComputationGraph> BuildGraph(TSetup& setup, const std::vector<std::tuple<ui32, i64, ui32>> items, + ui32 yieldPos, ui32 startIndex, bool dataWatermarks) { + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto structType = pgmBuilder.NewEmptyStructType(); + structType = pgmBuilder.NewStructType(structType, "key", + pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); + structType = pgmBuilder.NewStructType(structType, "time", + pgmBuilder.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id)); + structType = pgmBuilder.NewStructType(structType, "sum", + pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); + auto keyIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("key"); + auto timeIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("time"); + auto sumIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("sum"); + + auto inStreamType = pgmBuilder.NewStreamType(structType); + + TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", inStreamType); + auto streamNode = inStream.Build(); + + ui64 hop = 10, interval = 30, delay = 20; + + auto pgmReturn = pgmBuilder.MultiHoppingCore( + TRuntimeNode(streamNode, false), + [&](TRuntimeNode item) { // keyExtractor + return pgmBuilder.Member(item, "key"); + }, + [&](TRuntimeNode item) { // timeExtractor + return pgmBuilder.Member(item, "time"); + }, + [&](TRuntimeNode item) { // init + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("sum", pgmBuilder.Member(item, "sum")); + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode item, TRuntimeNode state) { // update + auto add = pgmBuilder.AggrAdd( + pgmBuilder.Member(item, "sum"), + pgmBuilder.Member(state, "sum")); + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("sum", add); + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode state) { // save + return pgmBuilder.Member(state, "sum"); + }, + [&](TRuntimeNode savedState) { // load + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("sum", savedState); + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode state1, TRuntimeNode state2) { // merge + auto add = pgmBuilder.AggrAdd( + pgmBuilder.Member(state1, "sum"), + pgmBuilder.Member(state2, "sum")); + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("sum", add); + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode key, TRuntimeNode state, TRuntimeNode time) { // finish + Y_UNUSED(time); + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("key", key); + members.emplace_back("sum", pgmBuilder.Member(state, "sum")); + return pgmBuilder.NewStruct(members); + }, + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&hop, sizeof(hop))), // hop + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&interval, sizeof(interval))), // interval + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&delay, sizeof(delay))), // delay + pgmBuilder.NewDataLiteral<bool>(dataWatermarks), // dataWatermarks + pgmBuilder.NewDataLiteral<bool>(false) + ); + + auto graph = setup.BuildGraph(pgmReturn, {streamNode}); + + TUnboxedValueVector streamItems; + for (size_t i = 0; i < items.size(); ++i) { + NUdf::TUnboxedValue* itemsPtr; + auto structValues = graph->GetHolderFactory().CreateDirectArrayHolder(3, itemsPtr); + itemsPtr[keyIndex] = NUdf::TUnboxedValuePod(std::get<0>(items[i])); + itemsPtr[timeIndex] = NUdf::TUnboxedValuePod(std::get<1>(items[i])); + itemsPtr[sumIndex] = NUdf::TUnboxedValuePod(std::get<2>(items[i])); + streamItems.push_back(std::move(structValues)); + } + + auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex)); + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue)); + return graph; + } +} + +Y_UNIT_TEST_SUITE(TMiniKQLMultiHoppingSaveLoadTest) { + void TestWithSaveLoadImpl( + const std::vector<std::tuple<ui32, i64, ui32>> input, + const std::vector<std::tuple<ui32, ui32>> expected, + bool withTraverse, + bool dataWatermarks) + { + TScopedAlloc alloc(__LOCATION__); + + for (ui32 yieldPos = 0; yieldPos < input.size(); ++yieldPos) { + std::vector<std::tuple<ui32, ui32>> result; + + TSetup setup1(alloc); + auto graph1 = BuildGraph(setup1, input, yieldPos, 0, dataWatermarks); + auto root1 = graph1->GetValue(); + + NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok; + while (status == NUdf::EFetchStatus::Ok) { + NUdf::TUnboxedValue val; + status = root1.Fetch(val); + if (status == NUdf::EFetchStatus::Ok) { + result.emplace_back(val.GetElement(0).Get<ui32>(), val.GetElement(1).Get<ui32>()); + } + } + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield); + + TString graphState; + if (withTraverse) { + SaveGraphState(&root1, 1, 0ULL, graphState); + } else { + graphState = graph1->SaveGraphState(); + } + + TSetup setup2(alloc); + auto graph2 = BuildGraph(setup2, input, -1, yieldPos, dataWatermarks); + NUdf::TUnboxedValue root2; + if (withTraverse) { + root2 = graph2->GetValue(); + LoadGraphState(&root2, 1, 0ULL, graphState); + } else { + graph2->LoadGraphState(graphState); + root2 = graph2->GetValue(); + } + + status = NUdf::EFetchStatus::Ok; + while (status == NUdf::EFetchStatus::Ok) { + NUdf::TUnboxedValue val; + status = root2.Fetch(val); + if (status == NUdf::EFetchStatus::Ok) { + result.emplace_back(val.GetElement(0).Get<ui32>(), val.GetElement(1).Get<ui32>()); + } + } + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish); + + auto sortedExpected = expected; + std::sort(result.begin(), result.end()); + std::sort(sortedExpected.begin(), sortedExpected.end()); + UNIT_ASSERT_EQUAL(result, sortedExpected); + } + } + + const std::vector<std::tuple<ui32, i64, ui32>> input1 = { + // Group; Time; Value + {2, 1, 2}, + {1, 1, 2}, + {2, 2, 3}, + {1, 2, 3}, + {2, 15, 4}, + {1, 15, 4}, + {2, 23, 6}, + {1, 23, 6}, + {2, 24, 5}, + {1, 24, 5}, + {2, 25, 7}, + {1, 25, 7}, + {2, 40, 2}, + {1, 40, 2}, + {2, 47, 1}, + {1, 47, 1}, + {2, 51, 6}, + {1, 51, 6}, + {2, 59, 2}, + {1, 59, 2}, + {2, 85, 8}, + {1, 85, 8} + }; + + const std::vector<std::tuple<ui32, ui32>> expected = { + {1, 8}, {1, 8}, {1, 8}, {1, 8}, + {1, 11}, {1, 11}, {1, 21}, {1, 22}, + {1, 27}, + {2, 8}, {2, 8}, {2, 8}, {2, 8}, + {2, 11}, {2, 11}, {2, 21}, + {2, 22}, {2, 27}}; + + Y_UNIT_TEST(Test1) { + TestWithSaveLoadImpl(input1, expected, true, false); + } + + Y_UNIT_TEST(Test2) { + TestWithSaveLoadImpl(input1, expected, false, false); + } +} + +} // namespace NMiniKQL +} // namespace NKikimr diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp new file mode 100644 index 00000000000..b47cfc222ef --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp @@ -0,0 +1,626 @@ +#include "../mkql_multihopping.h" +#include <ydb/library/yql/minikql/mkql_node.h> +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_program_builder.h> +#include <ydb/library/yql/minikql/mkql_function_registry.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h> +#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> +#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h> + +#include <library/cpp/testing/unittest/registar.h> + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + struct TInputItem { + ui32 Key = 0; + i64 Time = 0; + ui32 Val = 0; + }; + + struct TOutputItem { + ui32 Key = 0; + ui32 Val = 0; + ui64 Time = 0; + + constexpr bool operator==(const TOutputItem& rhs) const + { + return this->Key == rhs.Key && this->Val == rhs.Val && this->Time == rhs.Time; + } + }; + + struct TOutputGroup { + TOutputGroup(std::initializer_list<TOutputItem> items) : Items(items) {} + + std::vector<TOutputItem> Items; + }; + + std::vector<TOutputItem> Ordered(std::vector<TOutputItem> vec) { + std::sort(vec.begin(), vec.end(), [](auto l, auto r) { + return std::make_tuple(l.Key, l.Val, l.Time) < std::make_tuple(r.Key, r.Val, r.Time); + }); + return vec; + } + + IOutputStream &operator<<(IOutputStream &output, std::vector<TOutputItem> items) { + output << "["; + for (ui32 i = 0; i < items.size(); ++i) { + output << "(" << items.at(i).Key << ";" << items.at(i).Val << ";" << items.at(i).Time << ")"; + if (i != items.size() - 1) + output << ","; + } + output << "]"; + return output; + } + + TIntrusivePtr<IRandomProvider> CreateRandomProvider() { + return CreateDeterministicRandomProvider(1); + } + + TIntrusivePtr<ITimeProvider> CreateTimeProvider() { + return CreateDeterministicTimeProvider(10000000); + } + + TComputationNodeFactory GetAuxCallableFactory(TWatermark& watermark) { + return [&watermark](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* { + if (callable.GetType()->GetName() == "MyStream") { + return new TExternalComputationNode(ctx.Mutables); + } else if (callable.GetType()->GetName() == "MultiHoppingCore") { + return WrapMultiHoppingCore(callable, ctx, watermark); + } + + return GetBuiltinFactory()(callable, ctx); + }; + } + + struct TSetup { + TSetup(TScopedAlloc& alloc, TWatermark& watermark, bool watermarkMode = false) + : FunctionRegistry(CreateFunctionRegistry(CreateBuiltinRegistry())) + , RandomProvider(CreateRandomProvider()) + , TimeProvider(CreateTimeProvider()) + , StatsResgistry(CreateDefaultStatsRegistry()) + , Alloc(alloc) + , Watermark(watermark) + , WatermarkMode(watermarkMode) + { + Env.Reset(new TTypeEnvironment(Alloc)); + PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry)); + } + + THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) { + Explorer.Walk(pgm.GetNode(), *Env); + TComputationPatternOpts opts(Alloc.Ref(), *Env, GetAuxCallableFactory(Watermark), + FunctionRegistry.Get(), + NUdf::EValidateMode::None, NUdf::EValidatePolicy::Fail, "OFF", EGraphPerProcess::Multi, + StatsResgistry.Get()); + Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts); + TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider); + return Pattern->Clone(compOpts); + } + + TIntrusivePtr<IFunctionRegistry> FunctionRegistry; + TIntrusivePtr<IRandomProvider> RandomProvider; + TIntrusivePtr<ITimeProvider> TimeProvider; + IStatsRegistryPtr StatsResgistry; + + TScopedAlloc& Alloc; + THolder<TTypeEnvironment> Env; + THolder<TProgramBuilder> PgmBuilder; + + TExploringNodeVisitor Explorer; + IComputationPattern::TPtr Pattern; + + TWatermark& Watermark; + bool WatermarkMode; + }; + + struct TStream : public NUdf::TBoxedValue { + TStream(const TUnboxedValueVector& items, std::function<void()> fetchCallback, bool* yield) + : Items(items) + , FetchCallback(fetchCallback) + , yield(yield) {} + + private: + TUnboxedValueVector Items; + ui32 Index = 0; + std::function<void()> FetchCallback; + bool* yield; + + NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) final { + FetchCallback(); + if (*yield) { + return NUdf::EFetchStatus::Yield; + } + if (Index >= Items.size()) { + return NUdf::EFetchStatus::Finish; + } + result = Items[Index++]; + return NUdf::EFetchStatus::Ok; + } + }; + + THolder<IComputationGraph> BuildGraph( + TSetup& setup, + const std::vector<TInputItem> items, + std::function<void()> fetchCallback, + bool dataWatermarks, + bool* yield, + ui64 hop = 10, + ui64 interval = 30, + ui64 delay = 20) + { + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto structType = pgmBuilder.NewEmptyStructType(); + structType = pgmBuilder.NewStructType(structType, "key", + pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); + structType = pgmBuilder.NewStructType(structType, "time", + pgmBuilder.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id)); + structType = pgmBuilder.NewStructType(structType, "sum", + pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); + auto keyIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("key"); + auto timeIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("time"); + auto sumIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("sum"); + + auto inStreamType = pgmBuilder.NewStreamType(structType); + + TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "MyStream", inStreamType); + auto streamNode = inStream.Build(); + + auto pgmReturn = pgmBuilder.MultiHoppingCore( + TRuntimeNode(streamNode, false), + [&](TRuntimeNode item) { // keyExtractor + return pgmBuilder.Member(item, "key"); + }, + [&](TRuntimeNode item) { // timeExtractor + return pgmBuilder.Member(item, "time"); + }, + [&](TRuntimeNode item) { // init + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("sum", pgmBuilder.Member(item, "sum")); + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode item, TRuntimeNode state) { // update + auto add = pgmBuilder.AggrAdd( + pgmBuilder.Member(item, "sum"), + pgmBuilder.Member(state, "sum")); + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("sum", add); + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode state) { // save + return pgmBuilder.Member(state, "sum"); + }, + [&](TRuntimeNode savedState) { // load + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("sum", savedState); + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode state1, TRuntimeNode state2) { // merge + auto add = pgmBuilder.AggrAdd( + pgmBuilder.Member(state1, "sum"), + pgmBuilder.Member(state2, "sum")); + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("sum", add); + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode key, TRuntimeNode state, TRuntimeNode time) { // finish + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("key", key); + members.emplace_back("sum", pgmBuilder.Member(state, "sum")); + members.emplace_back("time", time); + return pgmBuilder.NewStruct(members); + }, + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&hop, sizeof(hop))), // hop + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&interval, sizeof(interval))), // interval + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&delay, sizeof(delay))), // delay + pgmBuilder.NewDataLiteral<bool>(dataWatermarks), + pgmBuilder.NewDataLiteral<bool>(setup.WatermarkMode) + ); + + auto graph = setup.BuildGraph(pgmReturn, {streamNode}); + + TUnboxedValueVector streamItems; + for (size_t i = 0; i < items.size(); ++i) { + NUdf::TUnboxedValue* itemsPtr; + auto structValues = graph->GetHolderFactory().CreateDirectArrayHolder(3, itemsPtr); + itemsPtr[keyIndex] = NUdf::TUnboxedValuePod(items.at(i).Key); + itemsPtr[timeIndex] = NUdf::TUnboxedValuePod(items.at(i).Time); + itemsPtr[sumIndex] = NUdf::TUnboxedValuePod(items.at(i).Val); + streamItems.push_back(std::move(structValues)); + } + + auto streamValue = NUdf::TUnboxedValuePod(new TStream(streamItems, fetchCallback, yield)); + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue)); + return graph; + } +} + +Y_UNIT_TEST_SUITE(TMiniKQLMultiHoppingTest) { + void TestImpl( + const std::vector<TInputItem>& input, + const std::vector<TOutputGroup>& expected, + bool dataWatermarks, + ui64 hop = 10, + ui64 interval = 30, + ui64 delay = 20, + std::function<void(ui32, TSetup&)> customCheck = [](ui32, TSetup&){}, + TWatermark* watermark = nullptr, + bool* yield = nullptr, + std::function<void()> fetch_callback= [](){}, + bool watermarkMode = false) + { + bool yield_clone = false; + if (!yield) { + yield = &yield_clone; + } + if (watermarkMode) { + dataWatermarks = false; + } + TWatermark watermark_clone{TInstant::Zero()}; + if (watermark == nullptr) { + watermark = &watermark_clone; + } + TScopedAlloc alloc(__LOCATION__); + TSetup setup1(alloc, *watermark, watermarkMode); + + ui32 curGroupId = 0; + std::vector<TOutputItem> curResult; + + auto check = [&curResult, &curGroupId, &expected, customCheck, &setup1, &fetch_callback]() { + fetch_callback(); + auto expectedItems = Ordered(expected.at(curGroupId).Items); // Add more empty lists at yield in expected + curResult = Ordered(curResult); + UNIT_ASSERT_EQUAL_C(curResult, expectedItems, "curGroup: " << curGroupId << " actual: " << curResult << " expected: " << expectedItems); + customCheck(curGroupId, setup1); + curGroupId++; + curResult.clear(); + }; + + auto graph1 = BuildGraph(setup1, input, check, dataWatermarks, yield, hop, interval, delay); + + auto root1 = graph1->GetValue(); + + NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok; + while (status == NUdf::EFetchStatus::Ok || status == NUdf::EFetchStatus::Yield) { + NUdf::TUnboxedValue val; + status = root1.Fetch(val); + if (status == NUdf::EFetchStatus::Ok) { + curResult.emplace_back(TOutputItem{val.GetElement(0).Get<ui32>(), val.GetElement(1).Get<ui32>(), val.GetElement(2).Get<ui64>()}); + } + } + + check(); + UNIT_ASSERT_EQUAL_C(curGroupId, expected.size(), "1: " << curGroupId << " 2: " << expected.size()); + } + + void TestWatermarksImpl( + const std::vector<TInputItem>& input, + const std::vector<TOutputGroup>& expected, + const std::vector<std::pair<ui64, TInstant>>& watermarks) + { + bool yield = false; + TWatermark watermark; + ui64 inp_index = 0; + ui64 pattern_index = 0; + auto avant_fetch = [&yield, &watermark, &watermarks, &inp_index, &pattern_index](){ + yield = false; + if (pattern_index >= watermarks.size()) { + return; + } + if (inp_index == watermarks[pattern_index].first) { + yield = true; + watermark.WatermarkIn = watermarks[pattern_index].second; + ++pattern_index; + } else { + ++inp_index; + } + }; + TestImpl(input, expected, false, 10, 30, 20, [](ui32, TSetup&){}, &watermark, &yield, avant_fetch, true); + } + + Y_UNIT_TEST(TestThrowWatermarkFromPast) { + const std::vector<TInputItem> input = { + // Group; Time; Value + {1, 101, 2}, + {1, 131, 3}, + {1, 200, 4}, + {1, 300, 5}, + {1, 400, 6} + }; + + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}) + }; + std::vector<std::pair<ui64, TInstant>> yield_pattern = { + {2, TInstant::MicroSeconds(20)}, + {3, TInstant::MicroSeconds(40)} + }; + TestWatermarksImpl(input, expected, yield_pattern); + } + + Y_UNIT_TEST(TestThrowWatermarkFromFuture) { + const std::vector<TInputItem> input = { + // Group; Time; Value + {1, 101, 2}, + {1, 131, 3}, + {1, 200, 4}, + {1, 300, 5}, + {1, 400, 6} + }; + + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}) + }; + std::vector<std::pair<ui64, TInstant>> yield_pattern = { + {2, TInstant::MicroSeconds(1000)}, + {3, TInstant::MicroSeconds(2000)} + }; + TestWatermarksImpl(input, expected, yield_pattern); + } + + Y_UNIT_TEST(TestWatermarkFlow1) { + const std::vector<TInputItem> input = { + // Group; Time; Value + {1, 101, 2}, + {1, 131, 3}, + {1, 200, 4}, + {1, 300, 5}, + {1, 400, 6} + }; + + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({{1, 2, 110},{1, 2, 120},{1, 2, 130}}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}) + }; + std::vector<std::pair<ui64, TInstant>> yield_pattern = { + {0, TInstant::MicroSeconds(100)}, + {3, TInstant::MicroSeconds(200)} + }; + TestWatermarksImpl(input, expected, yield_pattern); + } + + Y_UNIT_TEST(TestWatermarkFlow2) { + const std::vector<TInputItem> input = { + // Group; Time; Value + {1, 100, 2}, + {1, 105, 3}, + {1, 80, 4}, + {1, 107, 5}, + {1, 106, 6} + }; + + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({{1, 4, 90}, {1, 4, 100}, {1, 4, 110}}) + }; + std::vector<std::pair<ui64, TInstant>> yield_pattern = { + {0, TInstant::MicroSeconds(76)}, + }; + TestWatermarksImpl(input, expected, yield_pattern); + } + + Y_UNIT_TEST(TestWatermarkFlow3) { + const std::vector<TInputItem> input = { + // Group; Time; Value + {1, 90, 2}, + {1, 99, 3}, + {1, 80, 4}, + {1, 107, 5}, + {1, 106, 6} + }; + + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({{1, 4, 90}, {1, 9, 100}, {1, 9, 110}, {1, 5, 120}}) + }; + std::vector<std::pair<ui64, TInstant>> yield_pattern = { + {0, TInstant::MicroSeconds(76)}, + }; + TestWatermarksImpl(input, expected, yield_pattern); + } + + Y_UNIT_TEST(TestDataWatermarks) { + const std::vector<TInputItem> input = { + // Group; Time; Value + {1, 101, 2}, + {2, 101, 2}, + {1, 111, 3}, + {2, 140, 5}, + {2, 160, 1} + }; + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({{1, 2, 110}, {1, 5, 120}, {2, 2, 110}, {2, 2, 120}}), + TOutputGroup({{2, 2, 130}, {1, 5, 130}, {1, 3, 140}}), + TOutputGroup({{2, 5, 150}, {2, 5, 160}, {2, 6, 170}, {2, 1, 180}, {2, 1, 190}}), + }; + TestImpl(input, expected, true); + } + + Y_UNIT_TEST(TestDataWatermarksNoGarbage) { + const std::vector<TInputItem> input = { + // Group; Time; Value + {1, 100, 2}, + {2, 150, 1} + }; + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({{1, 2, 110}, {1, 2, 120}, {1, 2, 130}}), + TOutputGroup({{2, 1, 160}, {2, 1, 170}, {2, 1, 180}}), + }; + TestImpl(input, expected, true, 10, 30, 20, + [](ui32 curGroup, TSetup& setup) { + if (curGroup != 2) { + return; + } + + setup.StatsResgistry->ForEachStat([](const TStatKey& key, i64 value) { + if (key.GetName() == "MultiHop_KeysCount") { + UNIT_ASSERT_EQUAL_C(value, 1, "actual: " << value << " expected: " << 1); + } + }); + }); + } + + Y_UNIT_TEST(TestValidness1) { + const std::vector<TInputItem> input1 = { + // Group; Time; Value + {1, 101, 2}, + {2, 101, 2}, + {1, 111, 3}, + {2, 140, 5}, + {2, 160, 1} + }; + + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({{2, 2, 110}, {2, 2, 120}}), + TOutputGroup({{2, 2, 130}}), + TOutputGroup({{1, 2, 110}, {1, 5, 120}, {1, 5, 130}, {1, 3, 140}, {2, 5, 150}, + {2, 5, 160}, {2, 6, 170}, {2, 1, 190}, {2, 1, 180}}), + }; + TestImpl(input1, expected, false); + } + + Y_UNIT_TEST(TestValidness2) { + const std::vector<TInputItem> input = { + // Group; Time; Value + {2, 101, 2}, {1, 101, 2}, {2, 102, 3}, {1, 102, 3}, {2, 115, 4}, + {1, 115, 4}, {2, 123, 6}, {1, 123, 6}, {2, 124, 5}, {1, 124, 5}, + {2, 125, 7}, {1, 125, 7}, {2, 140, 2}, {1, 140, 2}, {2, 147, 1}, + {1, 147, 1}, {2, 151, 6}, {1, 151, 6}, {2, 159, 2}, {1, 159, 2}, + {2, 185, 8}, {1, 185, 8} + }; + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), + TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), + TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), + TOutputGroup({{1, 5, 110}, {1, 9, 120}, {2, 5, 110}, {2, 9, 120}}), + TOutputGroup({}), + TOutputGroup({}), TOutputGroup({}), + TOutputGroup({{2, 27, 130}, {1, 27, 130}}), + TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), + TOutputGroup({{2, 22, 140}, {2, 21, 150}, {2, 11, 160}, {1, 22, 140}, {1, 21, 150}, {1, 11, 160}}), + TOutputGroup({}), + TOutputGroup({{1, 11, 170}, {1, 8, 180}, {1, 8, 190}, {1, 8, 200}, {1, 8, 210}, {2, 11, 170}, + {2, 8, 180}, {2, 8, 190}, {2, 8, 200}, {2, 8, 210}}), + }; + + TestImpl(input, expected, true); + } + + Y_UNIT_TEST(TestValidness3) { + const std::vector<TInputItem> input = { + // Group; Time; Value + {1, 105, 1}, {1, 107, 4}, {2, 106, 3}, {1, 111, 7}, {1, 117, 3}, + {2, 110, 2}, {1, 108, 9}, {1, 121, 4}, {2, 107, 2}, {2, 141, 5}, + {1, 141, 10} + }; + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), + TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), + TOutputGroup({{1, 14, 110}, {2, 3, 110}}), + TOutputGroup({}), + TOutputGroup({{2, 7, 115}, {2, 2, 120}, {1, 21, 115}, {1, 10, 120}, {1, 7, 125}, {1, 4, 130}}), + TOutputGroup({}), + TOutputGroup({{1, 10, 145}, {1, 10, 150}, {2, 5, 145}, {2, 5, 150}}) + }; + + TestImpl(input, expected, true, 5, 10, 10); + } + + Y_UNIT_TEST(TestDelay) { + const std::vector<TInputItem> input = { + // Group; Time; Value + {1, 101, 3}, {1, 111, 5}, {1, 120, 7}, {1, 80, 9}, {1, 79, 11} + }; + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), + TOutputGroup({}), TOutputGroup({}), + TOutputGroup({{1, 12, 110}, {1, 8, 120}, {1, 15, 130}, {1, 12, 140}, {1, 7, 150}}) + }; + + TestImpl(input, expected, false); + } + + Y_UNIT_TEST(TestWindowsBeforeFirstElement) { + const std::vector<TInputItem> input = { + // Group; Time; Value + {1, 101, 2}, {1, 111, 3} + }; + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({{1, 2, 110}, {1, 5, 120}, {1, 5, 130}, {1, 3, 140}}) + }; + + TestImpl(input, expected, false); + } + + Y_UNIT_TEST(TestSubzeroValues) { + const std::vector<TInputItem> input = { + // Group; Time; Value + {1, 1, 2} + }; + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({{1, 2, 30}}), + }; + + TestImpl(input, expected, false); + } +} + +} // namespace NMiniKQL +} // namespace NKikimr diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_multimap_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multimap_ut.cpp new file mode 100644 index 00000000000..0c88e86c046 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multimap_ut.cpp @@ -0,0 +1,171 @@ +#include "mkql_computation_node_ut.h" +#include <ydb/library/yql/minikql/mkql_runtime_version.h> + +namespace NKikimr { +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLMultiMapTest) { + Y_UNIT_TEST_LLVM(TestOverList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto data3 = pb.NewDataLiteral<ui32>(3); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto list = pb.NewList(dataType, {data1, data2, data3}); + const auto pgmReturn = pb.MultiMap(list, + [&](TRuntimeNode item) { + return TRuntimeNode::TList{pb.Add(item, data1), item, pb.Mul(item, data2)}; + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 6); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestOverLazyList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto data3 = pb.NewDataLiteral<ui32>(3); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto list = pb.NewList(dataType, {data1, data2, data3}); + const auto pgmReturn = pb.MultiMap(pb.LazyList(list), + [&](TRuntimeNode item) { + return TRuntimeNode::TList{pb.Add(item, data1), item, pb.Mul(item, data2)}; + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 6); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestOverFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<ui32>(2); + const auto data3 = pb.NewDataLiteral<ui32>(3); + const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto list = pb.NewList(dataType, {data1, data2, data3}); + const auto pgmReturn = pb.Collect(pb.MultiMap(pb.ToFlow(list), + [&](TRuntimeNode item) { + return TRuntimeNode::TList{pb.Add(item, data1), item, pb.Mul(item, data2)}; + })); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 6); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u + Y_UNIT_TEST_LLVM(TestFlattenByNarrow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3}); + + const auto pgmReturn = pb.Collect(pb.NarrowMultiMap(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items[2U], items[1U], items[0U] }; } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 3); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } +#endif +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_rh_hash_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_rh_hash_ut.cpp new file mode 100644 index 00000000000..91cf6e3c77e --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_rh_hash_ut.cpp @@ -0,0 +1,271 @@ +#include "../mkql_rh_hash.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <unordered_map> +#include <unordered_set> + +namespace NKikimr { +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLRobinHoodHashTest) { + Y_UNIT_TEST(Map) { + TRobinHoodHashMap<i32> rh(sizeof(i64)); + std::unordered_map<i32, i64> h; + for (ui64 i = 0; i < 10000; ++i) { + auto k = i % 1000; + auto [it, inserted] = h.emplace(k, 0); + bool isNew; + auto iter = rh.Insert(k, isNew); + UNIT_ASSERT_VALUES_EQUAL(rh.GetKey(iter), k); + UNIT_ASSERT_VALUES_EQUAL(isNew, inserted); + it->second += i; + if (isNew) { + *(i64*)rh.GetMutablePayload(iter) = i; + rh.CheckGrow(); + } else { + *(i64*)rh.GetMutablePayload(iter) += i; + } + + UNIT_ASSERT_VALUES_EQUAL(h.size(), rh.GetSize()); + } + + for (auto it = rh.Begin(); it != rh.End(); rh.Advance(it)) { + if (!rh.IsValid(it)) { + continue; + } + + auto key = rh.GetKey(it); + auto hit = h.find(key); + UNIT_ASSERT(hit != h.end()); + UNIT_ASSERT_VALUES_EQUAL(*(i64*)rh.GetPayload(it), hit->second); + h.erase(key); + } + + UNIT_ASSERT(h.empty()); + } + + Y_UNIT_TEST(FixedMap) { + TRobinHoodHashFixedMap<i32, i64> rh; + std::unordered_map<i32, i64> h; + for (ui64 i = 0; i < 10000; ++i) { + auto k = i % 1000; + auto [it, inserted] = h.emplace(k, 0); + bool isNew; + auto iter = rh.Insert(k, isNew); + UNIT_ASSERT_VALUES_EQUAL(rh.GetKey(iter), k); + UNIT_ASSERT_VALUES_EQUAL(isNew, inserted); + it->second += i; + if (isNew) { + *(i64*)rh.GetMutablePayload(iter) = i; + rh.CheckGrow(); + } else { + *(i64*)rh.GetMutablePayload(iter) += i; + } + + UNIT_ASSERT_VALUES_EQUAL(h.size(), rh.GetSize()); + } + + for (auto it = rh.Begin(); it != rh.End(); rh.Advance(it)) { + if (!rh.IsValid(it)) { + continue; + } + + auto key = rh.GetKey(it); + auto hit = h.find(key); + UNIT_ASSERT(hit != h.end()); + UNIT_ASSERT_VALUES_EQUAL(*(i64*)rh.GetPayload(it), hit->second); + h.erase(key); + } + + UNIT_ASSERT(h.empty()); + } + + + Y_UNIT_TEST(Set) { + TRobinHoodHashSet<i32> rh; + std::unordered_set<i32> h; + for (ui64 i = 0; i < 10000; ++i) { + auto k = i % 1000; + auto[it, inserted] = h.emplace(k); + bool isNew; + auto iter = rh.Insert(k, isNew); + UNIT_ASSERT_VALUES_EQUAL(rh.GetKey(iter), k); + UNIT_ASSERT_VALUES_EQUAL(isNew, inserted); + if (isNew) { + rh.CheckGrow(); + } + + UNIT_ASSERT_VALUES_EQUAL(h.size(), rh.GetSize()); + } + + for (auto it = rh.Begin(); it != rh.End(); rh.Advance(it)) { + if (!rh.IsValid(it)) { + continue; + } + + auto key = rh.GetKey(it); + auto hit = h.find(key); + UNIT_ASSERT(hit != h.end()); + h.erase(key); + } + + UNIT_ASSERT(h.empty()); + } + + Y_UNIT_TEST(MapBatch) { + using THashTable = TRobinHoodHashMap<i32>; + THashTable rh(sizeof(i64)); + std::unordered_map<i32, i64> h; + std::array<TRobinHoodBatchRequestItem<i32>, PrefetchBatchSize> batch; + std::array<bool, PrefetchBatchSize> batchInserted; + std::array<ui64, PrefetchBatchSize> batchI; + ui32 batchLen = 0; + + auto processBatch = [&]() { + rh.BatchInsert({batch.data(), batchLen}, [&](size_t i, THashTable::iterator iter, bool isNew) { + UNIT_ASSERT_VALUES_EQUAL(isNew, batchInserted[i]); + UNIT_ASSERT_VALUES_EQUAL(rh.GetKey(iter), batch[i].GetKey()); + if (isNew) { + *(i64*)rh.GetMutablePayload(iter) = batchI[i]; + } else { + *(i64*)rh.GetMutablePayload(iter) += batchI[i]; + } + }); + + UNIT_ASSERT_VALUES_EQUAL(h.size(), rh.GetSize()); + }; + + for (ui64 i = 0; i < 10000; ++i) { + if (batchLen == batch.size()) { + processBatch(); + batchLen = 0; + } + + auto k = i % 1000; + auto [it, inserted] = h.emplace(k, 0); + batchI[batchLen] = i; + batchInserted[batchLen] = inserted; + batch[batchLen].ConstructKey(k); + ++batchLen; + it->second += i; + } + + processBatch(); + for (auto it = rh.Begin(); it != rh.End(); rh.Advance(it)) { + if (!rh.IsValid(it)) { + continue; + } + + auto key = rh.GetKey(it); + auto hit = h.find(key); + UNIT_ASSERT(hit != h.end()); + UNIT_ASSERT_VALUES_EQUAL(*(i64*)rh.GetPayload(it), hit->second); + h.erase(key); + } + + UNIT_ASSERT(h.empty()); + } + + Y_UNIT_TEST(FixedMapBatch) { + using THashTable = TRobinHoodHashFixedMap<i32, i64>; + THashTable rh(sizeof(i64)); + std::unordered_map<i32, i64> h; + std::array<TRobinHoodBatchRequestItem<i32>, PrefetchBatchSize> batch; + std::array<bool, PrefetchBatchSize> batchInserted; + std::array<ui64, PrefetchBatchSize> batchI; + ui32 batchLen = 0; + + auto processBatch = [&]() { + rh.BatchInsert({batch.data(), batchLen}, [&](size_t i, THashTable::iterator iter, bool isNew) { + UNIT_ASSERT_VALUES_EQUAL(isNew, batchInserted[i]); + UNIT_ASSERT_VALUES_EQUAL(rh.GetKey(iter), batch[i].GetKey()); + if (isNew) { + *(i64*)rh.GetMutablePayload(iter) = batchI[i]; + } else { + *(i64*)rh.GetMutablePayload(iter) += batchI[i]; + } + }); + + UNIT_ASSERT_VALUES_EQUAL(h.size(), rh.GetSize()); + }; + + for (ui64 i = 0; i < 10000; ++i) { + if (batchLen == batch.size()) { + processBatch(); + batchLen = 0; + } + + auto k = i % 1000; + auto [it, inserted] = h.emplace(k, 0); + batchI[batchLen] = i; + batchInserted[batchLen] = inserted; + batch[batchLen].ConstructKey(k); + ++batchLen; + it->second += i; + } + + processBatch(); + for (auto it = rh.Begin(); it != rh.End(); rh.Advance(it)) { + if (!rh.IsValid(it)) { + continue; + } + + auto key = rh.GetKey(it); + auto hit = h.find(key); + UNIT_ASSERT(hit != h.end()); + UNIT_ASSERT_VALUES_EQUAL(*(i64*)rh.GetPayload(it), hit->second); + h.erase(key); + } + + UNIT_ASSERT(h.empty()); + } + + Y_UNIT_TEST(SetBatch) { + using THashTable = TRobinHoodHashSet<i32>; + THashTable rh(sizeof(i64)); + std::unordered_set<i32> h; + std::array<TRobinHoodBatchRequestItem<i32>, PrefetchBatchSize> batch; + std::array<bool, PrefetchBatchSize> batchInserted; + ui32 batchLen = 0; + + auto processBatch = [&]() { + rh.BatchInsert({batch.data(), batchLen}, [&](size_t i, THashTable::iterator iter, bool isNew) { + UNIT_ASSERT_VALUES_EQUAL(isNew, batchInserted[i]); + UNIT_ASSERT_VALUES_EQUAL(rh.GetKey(iter), batch[i].GetKey()); + }); + + UNIT_ASSERT_VALUES_EQUAL(h.size(), rh.GetSize()); + }; + + for (ui64 i = 0; i < 10000; ++i) { + if (batchLen == batch.size()) { + processBatch(); + batchLen = 0; + } + + auto k = i % 1000; + auto [it, inserted] = h.emplace(k); + batchInserted[batchLen] = inserted; + batch[batchLen].ConstructKey(k); + ++batchLen; + } + + processBatch(); + for (auto it = rh.Begin(); it != rh.End(); rh.Advance(it)) { + if (!rh.IsValid(it)) { + continue; + } + + auto key = rh.GetKey(it); + auto hit = h.find(key); + UNIT_ASSERT(hit != h.end()); + h.erase(key); + } + + UNIT_ASSERT(h.empty()); + } +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_safe_circular_buffer_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_safe_circular_buffer_ut.cpp new file mode 100644 index 00000000000..e3545ed1fc2 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_safe_circular_buffer_ut.cpp @@ -0,0 +1,143 @@ +#include "../mkql_safe_circular_buffer.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <ydb/library/yql/public/udf/udf_value.h> + +namespace NKikimr { +using namespace NUdf; +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLSafeCircularBuffer) { + typedef TSafeCircularBuffer<TUnboxedValue> TBufUnboxed; + + Y_UNIT_TEST(TestUnboxedNoFailOnEmpty) { + TBufUnboxed bufferOptional(1, TUnboxedValuePod()); + TBufUnboxed buffer(1, TUnboxedValue::Void()); + UNIT_ASSERT(buffer.Get(0)); + UNIT_ASSERT(buffer.Get(1)); + UNIT_ASSERT(buffer.Get(3)); + UNIT_ASSERT(buffer.Get(-1)); + for (auto i = 0; i < 5; ++i) { + buffer.PopFront(); + } + } + + Y_UNIT_TEST(TestUnboxedNormalUsage) { + TBufUnboxed buffer(5, TUnboxedValuePod()); + buffer.PushBack(TUnboxedValue::Embedded("It")); + UNIT_ASSERT_EQUAL(buffer.Get(0).AsStringRef(), "It"); + buffer.PushBack(TUnboxedValue::Embedded("is")); + UNIT_ASSERT_EQUAL(buffer.Get(0).AsStringRef(), "It"); + buffer.PushBack(TUnboxedValue::Embedded("funny")); + UNIT_ASSERT_EQUAL(buffer.UsedSize(), 3); + UNIT_ASSERT_EQUAL(buffer.Get(0).AsStringRef(), "It"); + UNIT_ASSERT_EQUAL(buffer.Get(2).AsStringRef(), "funny"); + UNIT_ASSERT(!buffer.Get(3)); + buffer.PopFront(); + UNIT_ASSERT_EQUAL(buffer.Get(0).AsStringRef(), "is"); + UNIT_ASSERT_EQUAL(buffer.Get(1).AsStringRef(), "funny"); + buffer.PushBack(TUnboxedValue::Embedded("bunny")); + UNIT_ASSERT_EQUAL(buffer.Get(1).AsStringRef(), "funny"); + UNIT_ASSERT_EQUAL(buffer.Get(2).AsStringRef(), "bunny"); + UNIT_ASSERT(!buffer.Get(3)); + buffer.PopFront(); + UNIT_ASSERT_EQUAL(buffer.Get(0).AsStringRef(), "funny"); + UNIT_ASSERT_EQUAL(buffer.Get(1).AsStringRef(), "bunny"); + UNIT_ASSERT_EQUAL(buffer.UsedSize(), 2); + buffer.PopFront(); + UNIT_ASSERT_EQUAL(buffer.Get(0).AsStringRef(), "bunny"); + UNIT_ASSERT_EQUAL(buffer.UsedSize(), 1); + for (auto i = 0; i < 3; ++i) { + buffer.PopFront(); + UNIT_ASSERT_EQUAL(buffer.UsedSize(), 0); + UNIT_ASSERT(!buffer.Get(0)); + } + } + + Y_UNIT_TEST(TestOverflowNoInitSize) { + TBufUnboxed buffer(3, TUnboxedValuePod(), 0); + buffer.PushBack(TUnboxedValue::Embedded("1")); + buffer.PushBack(TUnboxedValue::Embedded("2")); + buffer.PushBack(TUnboxedValue::Embedded("3")); + UNIT_ASSERT_EXCEPTION(buffer.PushBack(TUnboxedValue::Embedded("4")), yexception); + } + + Y_UNIT_TEST(TestOverflowWithInitSize) { + TBufUnboxed buffer(3, TUnboxedValuePod(), 3); + buffer.PopFront(); + buffer.PushBack(TUnboxedValue::Embedded("1")); + buffer.PopFront(); + buffer.PushBack(TUnboxedValue::Embedded("2")); + UNIT_ASSERT_EQUAL(buffer.UsedSize(), 3); + UNIT_ASSERT_EQUAL(buffer.Get(1).AsStringRef(), "1"); + UNIT_ASSERT_EXCEPTION(buffer.PushBack(TUnboxedValue::Embedded("3")), yexception); + } + + Y_UNIT_TEST(TestOverflowOnEmpty) { + TBufUnboxed buffer(0, TUnboxedValuePod()); + buffer.PopFront(); + UNIT_ASSERT_EXCEPTION(buffer.PushBack(TUnboxedValue::Embedded("1")), yexception); + } + + Y_UNIT_TEST(TestUnbounded) { + TBufUnboxed buffer({}, TUnboxedValuePod(), 3); + for (size_t i = 0; i < 100; ++i) { + buffer.PushBack(TUnboxedValue::Embedded(ToString(i))); + } + + UNIT_ASSERT(!buffer.Get(0)); + UNIT_ASSERT(!buffer.Get(1)); + UNIT_ASSERT(!buffer.Get(2)); + + for (size_t i = 0; i < 100; ++i) { + UNIT_ASSERT_EQUAL(TStringBuf(buffer.Get(i + 3).AsStringRef()), ToString(i)); + } + + for (size_t i = 0; i < 100; ++i) { + buffer.PopFront(); + } + + UNIT_ASSERT_EQUAL(buffer.UsedSize(), 3); + UNIT_ASSERT_EQUAL(TStringBuf(buffer.Get(0).AsStringRef()), ToString(97)); + UNIT_ASSERT_EQUAL(TStringBuf(buffer.Get(1).AsStringRef()), ToString(98)); + UNIT_ASSERT_EQUAL(TStringBuf(buffer.Get(2).AsStringRef()), ToString(99)); + + buffer.PopFront(); + buffer.PopFront(); + buffer.PopFront(); + UNIT_ASSERT_EQUAL(buffer.UsedSize(), 0); + UNIT_ASSERT_EQUAL(buffer.Size(), 103); + } + + Y_UNIT_TEST(TestUnboxedFewCycles) { + const auto multFillLevel = 0.6; + const auto multPopLevel = 0.5; + const auto initSize = 10; + const auto iterationCount = 4; + TBufUnboxed buffer(initSize, NUdf::TUnboxedValuePod()); + unsigned lastDirectIndex = 0; + unsigned lastChecked = 0; + for (unsigned iteration = 0; iteration < iterationCount; ++iteration) { + for (auto i = 0; i < multFillLevel * initSize; ++i) { + buffer.PushBack(NUdf::TUnboxedValuePod(++lastDirectIndex)); + } + UNIT_ASSERT(buffer.UsedSize() > 0); + UNIT_ASSERT_EQUAL(buffer.Get(buffer.UsedSize() - 1).Get<unsigned>(), lastDirectIndex); + for (auto i = 0; i < multPopLevel * initSize; ++i) { + auto curUnboxed = buffer.Get(0); + auto cur = curUnboxed.Get<unsigned>(); + UNIT_ASSERT(lastChecked < cur); + lastChecked = cur; + buffer.PopFront(); + } + UNIT_ASSERT_EQUAL(buffer.UsedSize(), iteration + 1); + } + UNIT_ASSERT_EQUAL(buffer.UsedSize(), lastDirectIndex - lastChecked); + UNIT_ASSERT_EQUAL(buffer.Get(0).Get<unsigned>(), lastChecked + 1); + UNIT_ASSERT_EQUAL(buffer.Get(buffer.UsedSize() - 1).Get<unsigned>(), lastDirectIndex); + } +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_sort_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_sort_ut.cpp new file mode 100644 index 00000000000..49e98020e63 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_sort_ut.cpp @@ -0,0 +1,523 @@ +#include <ydb/library/yql/minikql/computation/mkql_computation_node_impl.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> +#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_node.h> +#include <ydb/library/yql/minikql/mkql_program_builder.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> + +#include <ydb/library/yql/utils/sort.h> + +#include "mkql_computation_node_ut.h" + +#include <random> +#include <ctime> +#include <algorithm> + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + +template <bool LLVM> +TRuntimeNode MakeStream(TSetup<LLVM>& setup) { + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + TCallableBuilder callableBuilder(*setup.Env, "TestYieldStream", + pgmBuilder.NewStreamType( + pgmBuilder.NewStructType({ + {TStringBuf("a"), pgmBuilder.NewDataType(NUdf::EDataSlot::Uint64)}, + {TStringBuf("b"), pgmBuilder.NewDataType(NUdf::EDataSlot::String)} + }) + ) + ); + + return TRuntimeNode(callableBuilder.Build(), false); +} + +TRuntimeNode StreamToString(TProgramBuilder& pgmBuilder, TRuntimeNode stream) { + return pgmBuilder.Condense(stream, + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("|"), + [&] (TRuntimeNode, TRuntimeNode) { return pgmBuilder.NewDataLiteral<bool>(false); }, + [&] (TRuntimeNode item, TRuntimeNode state) { + auto str = pgmBuilder.Concat( + pgmBuilder.Concat( + pgmBuilder.ToString(pgmBuilder.Member(item, "a")), + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("-") + ), + pgmBuilder.Member(item, "b") + ); + + return pgmBuilder.Concat(pgmBuilder.Concat(state, str), + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("|")); + } + ); +} + +} // namespace + +Y_UNIT_TEST_SUITE(TMiniKQLSortTest) { + Y_UNIT_TEST_LLVM(TestStreamSort) { + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto stream = MakeStream(setup); + std::vector<TRuntimeNode> order { + pgmBuilder.NewDataLiteral<bool>(true), + pgmBuilder.NewDataLiteral<bool>(false) + }; + auto sort = pgmBuilder.Sort(stream, pgmBuilder.NewTuple(order), + [&pgmBuilder](TRuntimeNode item) { + std::vector<TRuntimeNode> keys { + pgmBuilder.Member(item, "a"), + pgmBuilder.Member(item, "b") + }; + + return pgmBuilder.NewTuple(keys); + }); + + auto pgmResult = StreamToString(pgmBuilder, sort); + + auto graph = setup.BuildGraph(pgmResult); + auto value = graph->GetValue(); + + + NUdf::TUnboxedValue result; + auto yieldCount = 0U; + auto status = NUdf::EFetchStatus::Ok; + while (status != NUdf::EFetchStatus::Finish) { + status = value.Fetch(result); + if (status == NUdf::EFetchStatus::Yield) { + ++yieldCount; + } + } + + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish); + UNIT_ASSERT_EQUAL(yieldCount, 3U); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), + "|0-8|0-4|0-11|0-0|1-9|1-6|1-13|1-1|2-7|2-2|2-14|2-10|"); + } + + Y_UNIT_TEST_LLVM(TestFlowSortByLambdaComparator) { + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + const auto stream = MakeStream(setup); + + const auto sort = + pgmBuilder.FromFlow(pgmBuilder.FlatMap( + pgmBuilder.Condense1(pgmBuilder.ToFlow(stream), + [&](TRuntimeNode item) { return pgmBuilder.AsList(item); }, + [&](TRuntimeNode, TRuntimeNode) { return pgmBuilder.NewDataLiteral<bool>(false); }, + [&](TRuntimeNode item, TRuntimeNode state) { return pgmBuilder.Append(state, item); } + ), + [&](TRuntimeNode list) { + return pgmBuilder.StableSort(list, + [&](TRuntimeNode left, TRuntimeNode right) { + return pgmBuilder.Or({ + pgmBuilder.AggrLess(pgmBuilder.Member(left, "a"), pgmBuilder.Member(right, "a")), + pgmBuilder.And({ + pgmBuilder.AggrEquals(pgmBuilder.Member(left, "a"), pgmBuilder.Member(right, "a")), + pgmBuilder.AggrGreater(pgmBuilder.Member(left, "b"), pgmBuilder.Member(right, "b")) + }) + }); + } + ); + } + )); + + const auto pgmResult = StreamToString(pgmBuilder, sort); + + const auto graph = setup.BuildGraph(pgmResult); + const auto value = graph->GetValue(); + + + NUdf::TUnboxedValue result; + auto yieldCount = 0U; + auto status = NUdf::EFetchStatus::Ok; + while (status != NUdf::EFetchStatus::Finish) { + status = value.Fetch(result); + if (status == NUdf::EFetchStatus::Yield) { + ++yieldCount; + } + } + + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish); + UNIT_ASSERT_EQUAL(yieldCount, 3U); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), + "|0-8|0-4|0-11|0-0|1-9|1-6|1-13|1-1|2-7|2-2|2-14|2-10|"); + } + + Y_UNIT_TEST_LLVM(TestListSort) { + const std::array<double, 10U> xxx = {{9E9, -HUGE_VAL, 0.003, HUGE_VAL, +3.1415, -0.003, -7898.8, -3.1415, 3673.0, 0.003}}; + + TSetup<LLVM> setup; + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + std::array<TRuntimeNode, 10U> data; + std::transform(xxx.cbegin(), xxx.cend(), data.begin(), + std::bind(&TProgramBuilder::NewDataLiteral<double>, std::ref(pgmBuilder), std::placeholders::_1 )); + + const auto type = pgmBuilder.NewDataType(NUdf::TDataType<double>::Id); + const auto list = pgmBuilder.NewList(type, data); + + const auto pgmReturn = pgmBuilder.Sort(list, pgmBuilder.NewDataLiteral<bool>(false), + std::bind(&TProgramBuilder::Abs, std::ref(pgmBuilder), std::placeholders::_1 )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto& result = graph->GetValue(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), xxx.size()); + + auto copy = xxx; + std::stable_sort(copy.begin(), copy.end(), [](double l, double r){ return std::abs(l) > std::abs(r); }); + + for (auto i = 0U; i < copy.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL(copy[i], result.GetElement(i).template Get<double>()); + } + } + + Y_UNIT_TEST_LLVM(TestListTop) { + TSetup<LLVM> setup; + + std::default_random_engine eng; + eng.seed(std::time(nullptr)); + + std::uniform_real_distribution<double> unifd(-999.0, +999.0); + std::uniform_real_distribution<ui64> unifi; + + constexpr ui64 total = 999ULL; + + std::array<TRuntimeNode, total> data; + std::vector<std::pair<double, ui64>> test; + test.reserve(total); + + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + std::generate_n(std::back_inserter(test), total, [&]() { return std::make_pair(unifd(eng), unifi(eng) % 100U);}); + + std::transform(test.cbegin(), test.cend(), data.begin(), [&](const auto& pair) { + return pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<double>(pair.first), pgmBuilder.NewDataLiteral<ui64>(pair.second)}); + }); + + const auto tupleType = pgmBuilder.NewTupleType({pgmBuilder.NewDataType(NUdf::TDataType<double>::Id), pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)}); + const auto order = pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<bool>(false), pgmBuilder.NewDataLiteral<bool>(true)}); + const auto list = pgmBuilder.NewList(tupleType, data); + const auto extractor = [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.NewTuple({pgmBuilder.Nth(item, 1U), pgmBuilder.Abs(pgmBuilder.Nth(item, 0U))}); }; + const auto n = 17ULL; + const auto limit = pgmBuilder.NewDataLiteral<ui64>(n); + + const auto pgmReturn = pgmBuilder.Top(list, limit, order, extractor); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto& value = graph->GetValue(); + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), n); + + const auto& iter = value.GetListIterator(); + + auto copy = test; + + const auto comp = [](const auto& l, const auto& r){ return l.second > r.second || (l.second == r.second && std::abs(l.first) < std::abs(r.first)); }; + NYql::FastNthElement(copy.begin(), copy.begin() + n - 1ULL, copy.end(), comp); + copy.resize(n); + + std::vector<std::pair<double, ui64>> res; + res.reserve(n); + + for (NUdf::TUnboxedValue item; iter.Next(item);) { + res.emplace_back(item.GetElement(0U).template Get<double>(), item.GetElement(1U).template Get<ui64>()); + } + + std::sort(copy.begin(), copy.end()); + std::sort(res.begin(), res.end()); + UNIT_ASSERT(copy == res); + } + + Y_UNIT_TEST_LLVM(TestStreamZeroTop) { + TSetup<LLVM> setup; + + constexpr ui64 total = 9ULL; + + std::array<TRuntimeNode, total> data; + + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + std::generate_n(data.begin(), total, [&]() { + return pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<ui64>(42ULL)}); + }); + + const auto tupleType = pgmBuilder.NewTupleType({pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)}); + const auto order = pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<bool>(false)}); + const auto list = pgmBuilder.NewList(tupleType, data); + const auto extractor = [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.NewTuple({pgmBuilder.Nth(item, 0U)}); }; + const auto limit = pgmBuilder.NewDataLiteral<ui64>(0ULL); + + const auto pgmReturn = pgmBuilder.Top(pgmBuilder.Iterator(list, {}), limit, order, extractor); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto& value = graph->GetValue(); + NUdf::TUnboxedValue stub; + UNIT_ASSERT_VALUES_EQUAL(value.Fetch(stub), NUdf::EFetchStatus::Finish); + } + + Y_UNIT_TEST_LLVM(TestListTopSort) { + TSetup<LLVM> setup; + + std::default_random_engine eng; + eng.seed(std::time(nullptr)); + + std::uniform_real_distribution<double> unifd(-999.0, +999.0); + std::uniform_real_distribution<ui64> unifi; + + constexpr ui64 total = 999ULL; + + std::array<TRuntimeNode, total> data; + std::vector<std::pair<double, ui64>> test; + test.reserve(total); + + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + std::generate_n(std::back_inserter(test), total, [&]() { return std::make_pair(unifd(eng), unifi(eng) % 100U);}); + + std::transform(test.cbegin(), test.cend(), data.begin(), [&](const auto& pair) { + return pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<double>(pair.first), pgmBuilder.NewDataLiteral<ui64>(pair.second)}); + }); + + const auto tupleType = pgmBuilder.NewTupleType({pgmBuilder.NewDataType(NUdf::TDataType<double>::Id), pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)}); + const auto order = pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<bool>(false), pgmBuilder.NewDataLiteral<bool>(true)}); + const auto list = pgmBuilder.NewList(tupleType, data); + const auto extractor = [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.NewTuple({pgmBuilder.Nth(item, 1U), pgmBuilder.Abs(pgmBuilder.Nth(item, 0U))}); }; + const auto n = 17ULL; + const auto limit = pgmBuilder.NewDataLiteral<ui64>(n); + + const auto pgmReturn = pgmBuilder.TopSort(list, limit, order, extractor); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto& value = graph->GetValue(); + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), n); + + const auto& iter = value.GetListIterator(); + + auto copy = test; + + const auto comp = [](const auto& l, const auto& r){ return l.second > r.second || (l.second == r.second && std::abs(l.first) < std::abs(r.first)); }; + NYql::FastPartialSort(copy.begin(), copy.begin() + n, copy.end(), comp); + copy.resize(n); + + std::vector<std::pair<double, ui64>> res; + res.reserve(n); + + for (NUdf::TUnboxedValue item; iter.Next(item);) { + res.emplace_back(item.GetElement(0U).template Get<double>(), item.GetElement(1U).template Get<ui64>()); + } + + UNIT_ASSERT(copy == res); + } + + Y_UNIT_TEST_LLVM(TestStreamTop) { + TSetup<LLVM> setup; + + std::default_random_engine eng; + eng.seed(std::time(nullptr)); + + std::uniform_real_distribution<double> unifd(-999.0, +999.0); + std::uniform_real_distribution<ui64> unifi; + + constexpr ui64 total = 999ULL; + + std::array<TRuntimeNode, total> data; + std::vector<std::pair<double, ui64>> test; + test.reserve(total); + + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + std::generate_n(std::back_inserter(test), total, [&]() { return std::make_pair(unifd(eng), unifi(eng) % 100U);}); + + std::transform(test.cbegin(), test.cend(), data.begin(), [&](const auto& pair) { + return pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<double>(pair.first), pgmBuilder.NewDataLiteral<ui64>(pair.second)}); + }); + + const auto tupleType = pgmBuilder.NewTupleType({pgmBuilder.NewDataType(NUdf::TDataType<double>::Id), pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)}); + const auto order = pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<bool>(true), pgmBuilder.NewDataLiteral<bool>(false)}); + const auto list = pgmBuilder.NewList(tupleType, data); + const auto extractor = [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.NewTuple({pgmBuilder.Nth(item, 1U), pgmBuilder.Abs(pgmBuilder.Nth(item, 0U))}); }; + const auto n = 17ULL; + const auto limit = pgmBuilder.NewDataLiteral<ui64>(n); + + const auto pgmReturn = pgmBuilder.Top(pgmBuilder.Iterator(list, {}), limit, order, extractor); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto& value = graph->GetValue(); + + auto copy = test; + + const auto comp = [](const auto& l, const auto& r){ return l.second < r.second || (l.second == r.second && std::abs(l.first) > std::abs(r.first)); }; + NYql::FastNthElement(copy.begin(), copy.begin() + n - 1ULL, copy.end(), comp); + copy.resize(n); + + std::vector<std::pair<double, ui64>> res; + res.reserve(n); + + for (NUdf::TUnboxedValue item; NUdf::EFetchStatus::Ok == value.Fetch(item);) { + res.emplace_back(item.GetElement(0U).template Get<double>(), item.GetElement(1U).template Get<ui64>()); + } + + UNIT_ASSERT_VALUES_EQUAL(res.size(), n); + + std::sort(copy.begin(), copy.end()); + std::sort(res.begin(), res.end()); + UNIT_ASSERT(copy == res); + } + + Y_UNIT_TEST_LLVM(TestStreamTopSort) { + TSetup<LLVM> setup; + + std::default_random_engine eng; + eng.seed(std::time(nullptr)); + + std::uniform_real_distribution<double> unifd(-999.0, +999.0); + std::uniform_real_distribution<ui64> unifi; + + constexpr ui64 total = 999ULL; + + std::array<TRuntimeNode, total> data; + std::vector<std::pair<double, ui64>> test; + test.reserve(total); + + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + std::generate_n(std::back_inserter(test), total, [&]() { return std::make_pair(unifd(eng), unifi(eng) % 100U);}); + + std::transform(test.cbegin(), test.cend(), data.begin(), [&](const auto& pair) { + return pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<double>(pair.first), pgmBuilder.NewDataLiteral<ui64>(pair.second)}); + }); + + const auto tupleType = pgmBuilder.NewTupleType({pgmBuilder.NewDataType(NUdf::TDataType<double>::Id), pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)}); + const auto order = pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<bool>(true), pgmBuilder.NewDataLiteral<bool>(false)}); + const auto list = pgmBuilder.NewList(tupleType, data); + const auto extractor = [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.NewTuple({pgmBuilder.Nth(item, 1U), pgmBuilder.Abs(pgmBuilder.Nth(item, 0U))}); }; + const auto n = 17ULL; + const auto limit = pgmBuilder.NewDataLiteral<ui64>(n); + + const auto pgmReturn = pgmBuilder.TopSort(pgmBuilder.Iterator(list, {}), limit, order, extractor); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto& value = graph->GetValue(); + + auto copy = test; + + const auto comp = [](const auto& l, const auto& r){ return l.second < r.second || (l.second == r.second && std::abs(l.first) > std::abs(r.first)); }; + NYql::FastPartialSort(copy.begin(), copy.begin() + n, copy.end(), comp); + copy.resize(n); + + std::vector<std::pair<double, ui64>> res; + res.reserve(n); + + for (NUdf::TUnboxedValue item; NUdf::EFetchStatus::Ok == value.Fetch(item);) { + res.emplace_back(item.GetElement(0U).template Get<double>(), item.GetElement(1U).template Get<ui64>()); + } + + UNIT_ASSERT_VALUES_EQUAL(res.size(), n); + UNIT_ASSERT(copy == res); + } + + Y_UNIT_TEST_LLVM(TestStreamTopSortBySingleField) { + TSetup<LLVM> setup; + + std::default_random_engine eng; + eng.seed(std::time(nullptr)); + + std::uniform_real_distribution<ui64> unifi; + + constexpr ui64 total = 999ULL; + + std::array<TRuntimeNode, total> data; + std::vector<ui64> test; + test.reserve(total); + + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + std::generate_n(std::back_inserter(test), total, [&]() { return unifi(eng) % 100ULL; }); + + std::transform(test.cbegin(), test.cend(), data.begin(), [&](const ui64& v) { + return pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<ui64>(v)}); + }); + + const auto tupleType = pgmBuilder.NewTupleType({pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)}); + const auto order = pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<bool>(false)}); + const auto list = pgmBuilder.NewList(tupleType, data); + const auto extractor = [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.NewTuple({pgmBuilder.Nth(item, 0U)}); }; + const auto n = 17ULL; + const auto limit = pgmBuilder.NewDataLiteral<ui64>(n); + + const auto pgmReturn = pgmBuilder.TopSort(pgmBuilder.Iterator(list, {}), limit, order, extractor); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto& value = graph->GetValue(); + + auto copy = test; + + NYql::FastPartialSort(copy.begin(), copy.begin() + n, copy.end(), std::greater<ui64>()); + copy.resize(n); + + std::vector<ui64> res; + res.reserve(n); + + for (NUdf::TUnboxedValue item; NUdf::EFetchStatus::Ok == value.Fetch(item);) { + res.emplace_back(item.GetElement(0U).template Get<ui64>()); + } + + UNIT_ASSERT_VALUES_EQUAL(res.size(), n); + UNIT_ASSERT(copy == res); + } + + Y_UNIT_TEST_LLVM(TestFlowTopSortWithoutKey) { + TSetup<LLVM> setup; + + std::default_random_engine eng; + eng.seed(std::time(nullptr)); + + std::uniform_real_distribution<ui64> unifi; + + constexpr ui64 total = 99ULL; + + std::array<TRuntimeNode, total> data; + std::vector<ui64> test; + test.reserve(total); + + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + std::generate_n(std::back_inserter(test), total, [&]() { return unifi(eng) % 100ULL; }); + + std::transform(test.cbegin(), test.cend(), data.begin(), [&](const ui64& v) { + return pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<ui64>(v)}); + }); + + const auto tupleType = pgmBuilder.NewTupleType({pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)}); + const auto order = pgmBuilder.NewTuple({}); + const auto list = pgmBuilder.NewList(tupleType, data); + const auto extractor = [&pgmBuilder](TRuntimeNode) { return pgmBuilder.NewTuple({}); }; + const auto n = 17ULL; + const auto limit = pgmBuilder.NewDataLiteral<ui64>(n); + + const auto pgmReturn = pgmBuilder.FromFlow(pgmBuilder.TopSort(pgmBuilder.ToFlow(list), limit, order, extractor)); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto& value = graph->GetValue(); + + auto copy = test; + copy.resize(n); + + std::vector<ui64> res; + res.reserve(n); + + for (NUdf::TUnboxedValue item; NUdf::EFetchStatus::Ok == value.Fetch(item);) { + res.emplace_back(item.GetElement(0U).template Get<ui64>()); + } + + UNIT_ASSERT_VALUES_EQUAL(res.size(), n); + UNIT_ASSERT(copy == res); + } +} +} // NMiniKQL +} // NKikimr diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_switch_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_switch_ut.cpp new file mode 100644 index 00000000000..37472db9d60 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_switch_ut.cpp @@ -0,0 +1,276 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> + +namespace NKikimr { +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLSwitchTest) { + Y_UNIT_TEST_LLVM(TestStreamOfVariantsSwap) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1U); + const auto data2 = pb.NewDataLiteral<ui32>(2U); + const auto data3 = pb.NewDataLiteral<ui32>(3U); + + const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("123"); + const auto data5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("456"); + const auto data6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("789"); + + const auto intType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto strType = pb.NewDataType(NUdf::TDataType<char*>::Id); + + const auto varInType = pb.NewVariantType(pb.NewTupleType({intType, strType})); + + const auto var1 = pb.NewVariant(data1, 0U, varInType); + const auto var2 = pb.NewVariant(data2, 0U, varInType); + const auto var3 = pb.NewVariant(data3, 0U, varInType); + const auto var4 = pb.NewVariant(data4, 1U, varInType); + const auto var5 = pb.NewVariant(data5, 1U, varInType); + const auto var6 = pb.NewVariant(data6, 1U, varInType); + + const auto varOutType = pb.NewVariantType(pb.NewTupleType({strType, intType})); + + const auto list = pb.NewList(varInType, {var1, var2, var3, var4, var5, var6}); + + const auto pgmReturn = pb.Switch(pb.Iterator(list, {}), + {{{0U}, pb.NewStreamType(intType), std::nullopt}, {{1U}, pb.NewStreamType(strType), std::nullopt}}, + [&](ui32 index, TRuntimeNode stream) { + switch (index) { + case 0U: return pb.Map(stream, [&](TRuntimeNode item) { return pb.NewVariant(pb.ToString(item), 0U, varOutType); }); + case 1U: return pb.Map(stream, [&](TRuntimeNode item) { return pb.NewVariant(pb.StrictFromString(item, intType), 1U, varOutType); }); + } + Y_FAIL("Wrong case!"); + }, + 0ULL, + pb.NewStreamType(varOutType) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 0U); + UNBOXED_VALUE_STR_EQUAL(item, "1"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 0U); + UNBOXED_VALUE_STR_EQUAL(item, "2"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 0U); + UNBOXED_VALUE_STR_EQUAL(item, "3"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 123U); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 456U); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 789U); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestStreamOfVariantsTwoInOne) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1U); + const auto data2 = pb.NewDataLiteral<ui32>(2U); + const auto data3 = pb.NewDataLiteral<ui32>(3U); + + const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("123"); + const auto data5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("456"); + const auto data6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("789"); + + const auto intType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto strType = pb.NewDataType(NUdf::TDataType<char*>::Id); + + const auto varInType = pb.NewVariantType(pb.NewTupleType({intType, strType})); + + const auto var1 = pb.NewVariant(data1, 0U, varInType); + const auto var2 = pb.NewVariant(data2, 0U, varInType); + const auto var3 = pb.NewVariant(data3, 0U, varInType); + const auto var4 = pb.NewVariant(data4, 1U, varInType); + const auto var5 = pb.NewVariant(data5, 1U, varInType); + const auto var6 = pb.NewVariant(data6, 1U, varInType); + + const auto varOutType = pb.NewVariantType(pb.NewTupleType({strType, intType})); + + const auto list = pb.NewList(varInType, {var1, var2, var3, var4, var5, var6}); + + const auto pgmReturn = pb.Switch(pb.Iterator(list, {}), + {{{0U}, pb.NewStreamType(intType), 1U}, {{1U}, pb.NewStreamType(strType), std::nullopt}}, + [&](ui32 index, TRuntimeNode stream) { + switch (index) { + case 0U: return pb.Map(stream, [&](TRuntimeNode item) { return item; }); + case 1U: return pb.Map(stream, [&](TRuntimeNode item) { return pb.NewVariant(pb.StrictFromString(item, intType), 1U, varOutType); }); + } + Y_FAIL("Wrong case!"); + }, + 0ULL, + pb.NewStreamType(varOutType) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 1U); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 2U); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 3U); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 123U); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 456U); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 789U); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestFlowOfVariantsSwap) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1U); + const auto data2 = pb.NewDataLiteral<ui32>(2U); + const auto data3 = pb.NewDataLiteral<ui32>(3U); + + const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("123"); + const auto data5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("456"); + const auto data6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("789"); + + const auto intType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto strType = pb.NewDataType(NUdf::TDataType<char*>::Id); + + const auto varInType = pb.NewVariantType(pb.NewTupleType({intType, strType})); + + const auto var1 = pb.NewVariant(data1, 0U, varInType); + const auto var2 = pb.NewVariant(data2, 0U, varInType); + const auto var3 = pb.NewVariant(data3, 0U, varInType); + const auto var4 = pb.NewVariant(data4, 1U, varInType); + const auto var5 = pb.NewVariant(data5, 1U, varInType); + const auto var6 = pb.NewVariant(data6, 1U, varInType); + + const auto varOutType = pb.NewVariantType(pb.NewTupleType({strType, intType})); + + const auto list = pb.NewList(varInType, {var1, var2, var3, var4, var5, var6}); + + const auto pgmReturn = pb.FromFlow(pb.Switch(pb.ToFlow(list), + {{{0U}, pb.NewFlowType(intType), std::nullopt}, {{1U}, pb.NewFlowType(strType), std::nullopt}}, + [&](ui32 index, TRuntimeNode stream) { + switch (index) { + case 0U: return pb.Map(stream, [&](TRuntimeNode item) { return pb.NewVariant(pb.ToString(item), 0U, varOutType); }); + case 1U: return pb.Map(stream, [&](TRuntimeNode item) { return pb.NewVariant(pb.StrictFromString(item, intType), 1U, varOutType); }); + } + Y_FAIL("Wrong case!"); + }, + 0ULL, + pb.NewFlowType(varOutType) + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 0U); + UNBOXED_VALUE_STR_EQUAL(item, "1"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 0U); + UNBOXED_VALUE_STR_EQUAL(item, "2"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 0U); + UNBOXED_VALUE_STR_EQUAL(item, "3"); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 123U); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 456U); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 789U); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestFlowOfVariantsTwoInOne) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1U); + const auto data2 = pb.NewDataLiteral<ui32>(2U); + const auto data3 = pb.NewDataLiteral<ui32>(3U); + + const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("123"); + const auto data5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("456"); + const auto data6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("789"); + + const auto intType = pb.NewDataType(NUdf::TDataType<ui32>::Id); + const auto strType = pb.NewDataType(NUdf::TDataType<char*>::Id); + + const auto varInType = pb.NewVariantType(pb.NewTupleType({intType, strType})); + + const auto var1 = pb.NewVariant(data1, 0U, varInType); + const auto var2 = pb.NewVariant(data2, 0U, varInType); + const auto var3 = pb.NewVariant(data3, 0U, varInType); + const auto var4 = pb.NewVariant(data4, 1U, varInType); + const auto var5 = pb.NewVariant(data5, 1U, varInType); + const auto var6 = pb.NewVariant(data6, 1U, varInType); + + const auto varOutType = pb.NewVariantType(pb.NewTupleType({strType, intType})); + + const auto list = pb.NewList(varInType, {var1, var2, var3, var4, var5, var6}); + + const auto pgmReturn = pb.FromFlow(pb.Switch(pb.ToFlow(list), + {{{0U}, pb.NewFlowType(intType), 1U}, {{1U}, pb.NewFlowType(strType), std::nullopt}}, + [&](ui32 index, TRuntimeNode stream) { + switch (index) { + case 0U: return pb.Map(stream, [&](TRuntimeNode item) { return item; }); + case 1U: return pb.Map(stream, [&](TRuntimeNode item) { return pb.NewVariant(pb.StrictFromString(item, intType), 1U, varOutType); }); + } + Y_FAIL("Wrong case!"); + }, + 0ULL, + pb.NewFlowType(varOutType) + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 1U); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 2U); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 3U); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 123U); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 456U); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 789U); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp new file mode 100644 index 00000000000..75bf0fe352a --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp @@ -0,0 +1,549 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> + +#include <util/random/shuffle.h> +#include <map> +#include <optional> + +namespace NKikimr::NMiniKQL { + +static const TStringBuf data[] = { + "13d49d4db08e57d645fe4d44bbed4738f386af6e9e742cf186961063feb9919b", + "14d285e88582d87c41d3e6d2e9352686d0363ea74a297fe02f901f18c19978a3", + "1795ad46329c4fc6b3355dc22d252c5fe390a971ddf009b54fdeceb93d3b8930", + "18042e88fb4cf6b09cb8e6c5588ae525fc7a37bd2248a857d83ac1d1dcdf0a64", + "1b30b154ac814f7e4ed7e7488e037d781b78fbc336cac027f4c301ad9368514e", + "1a0b94cebdc038bb293575af4f6954e9dbf19801e581ad03be90b4aef36347d7", + "1c9ac5b87de7d68efae1bdf1ad47e58d28a6a70e966f19899798c61b2a65b6e2", + "1618c1e3d9dbc3edaccb7934eca55d2d96cb59d2633655f57401dba99deec3ef", + "1bd7a6ff86a1940283e202b142ecba685fea86f93f2aafad8cd37d80582aca95", + "0fba3f2f741b0579a3eec906f5341e0556fbd74088fcdfbe776bd6122fa81681", + "19768b3228cef7a82e0f1c367d78c42596fa511c735bd85d7cafca0965045562", + "1a9c0a14272795d7ad39a4725754c3f1d013a761c41fba16e06ae247833fd42b", + "1562ce72ff7229866723e81e35db30d08c8b8dc7b7d076cff787f157d70763e6", + "0faf214bafe219413618fdf186bb9290e6a610755d90947cd686b72899e78445", + "14f3fe97da837197b98f6303ac5aa1b6f34bffe9841fe65f084a27f4bd4ced8a", + "198c0706af7107ababebf1500875ba64508519b21aa534d0f55e8a32e394799d", + "1bb66a4593b77b1650a4a530bae30e454c2815477769d67fe6c6b337ae4acafe", + "0f67ef1ca6ef6b2d699dfac0360e8f24dc8960428cff058fe285d63ab55ef6d3", + "1097009fe853793802120394fbb6404df82725240d410c69375e5a53ec5395b8", + "1b1262275eae8a055732253e8d717c19ebde386b25e51dd546413e7ee997c5e1", + "1c4a73588541a0c869b4ee27c32cc4218f3c8db13713c61cedc387336a2720c9", + "1c73442f0ac53d8b38f779231680fab806a6cb9c86d25d9db5fa67c0ebf8e803", + "19152f0c06baf7962ca287a303b85437f321d725985f1586ac8358bdb6a0df63", + "13436f337815f5929559e6f621b850ed60b36f23ce9d8d06db981b70d40ad3db", + "298268d866eea5d6fcae470fdbb6d7787d73ab1e50b8126d6452d81264fbdafd", + "1a67b4e4c213baa140c5a00352cdbc9256b4e2fe81482c380b08ebe2e6b76e1b", + "19824d2008be54e35a0e2a9d2df9746e96f73367518b111695e1c3857966c241", + "2997c49ed21482d30b8ef89bd26bfdb6384dda6825032145fe0a3ad9d2f2a7e3", + "137ccc1d4ab00210bd9af5ee875cb799bd818f4803470abca68a9655ea73be01", + "12d4cf2eb41c90ede84bece72f76e97d7d0144c45341a0176f656b295cb838c3", + "11d02da4f449e6aeee4f213409baed6eaab35496688d743991870ba093676c44", + "163fb1ef04a1453a44fb897014287f7ceefe0b553d99718f986ada32cec6ca29", + "16f579a7eda4d7f5cde29724bf35e1b36e95fbeb72914ba2ba8f19192b92dab7", + "0f60c1387bf29d8d81174bd43c136e75f0f33b8b4d2712de0cc3a23f04fac76e", + "0f83662d3b4cc9aaa0f76c8801d2d32909c050286d514acc108b6c3d9964679b", + "1a30b7c4bf1c4eaaa92802cb90a27b5073d4a5ec095271490df8443b8f5df88f", + "105af591b517f126c985f54e833d61907ff49945ab123a606caa6d9bda0e3d66", + "1a5196fdfc1b81974905a66e6f1ff31403fc84b4d84effde521e848153f42e10", + "17d6cb5ba9489d8397cb1e1d054e45cff6c7225aeeba5c9e76cacd9da6c9a0c1", + "127ab4e2169329082bdd546e94c4fb6643999b14a26e08eaa719689789767014", + "143883410f000b5f3ff4c6434b0654516e9502d0a50a2b3ecdc20c8d3e198915", + "16ccd345646dd3d39e6bd157b51513c1b319bd1f441199003052a74b2eddb53d", + "11e9f02dc56d575fac5a306a6e83f597ffda1bd81a01f13fdda059ab24d90892", + "13f75a9e662faea5fc93f0f83d363c42083345cdcc42f1b0d320d11409ef3052", + "18cca97e8c6ede52e0b7d8c53c85c0fac68f6d1b7c6622a4cebc21433e6d8eea", + "160d6b818fab5ad00a1e81e46888c5ff3e5f2c175c013ce17d88c31df4475aba", + "1c4d09dff19175af7fc0d8e8fd23e9288fc2839dedfc067dcf9f5a3e3a9d92aa", + "16e25b2a6eef4cde6879c20c94c4360604b9099c29e1abaf9fc079fe67cfcaac", + "2a577ab7e2541e2cc2cc20e6a76c4ea9b77501808db9c4045be82c680cf227d5", + "11b4753fd9cc33656dbd59769b3202b7f68bd067bf7f64bd54676f6f60366ef1", + "1932a0aecc4a569d7d3fbcdd329b92c0b4dbd870d6be48ec4f18285ab3183676", + "2a2e6b62a4383cb48ffbb69b2f356ceb0410593f5b5500142498692dec7c125f", +}; + + +Y_UNIT_TEST_SUITE(TMiniKQLToDictTest) { + Y_UNIT_TEST_LLVM(TestCompactUtf8Set) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + TVector<TRuntimeNode> items; + for (auto s: data) { + items.push_back(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(s)); + } + Shuffle(items.begin(), items.end()); + auto dataType = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id); + auto list = pb.NewList(dataType, items); + auto dict = pb.ToHashedDict(list, false, [](TRuntimeNode n) { return n; }, [&pb](TRuntimeNode /*n*/) { return pb.NewVoid(); }, true); + auto pgmReturn = pb.Contains(dict, items.front()); + + auto graph = setup.BuildGraph(pgmReturn); + auto res = graph->GetValue().template Get<bool>(); + UNIT_ASSERT_VALUES_EQUAL(res, true); + } + + Y_UNIT_TEST_LLVM(TestUtf8Set) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + TVector<TRuntimeNode> items; + for (auto s: data) { + items.push_back(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(s)); + } + Shuffle(items.begin(), items.end()); + auto dataType = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id); + auto list = pb.NewList(dataType, items); + auto dict = pb.ToHashedDict(list, false, [](TRuntimeNode n) { return n; }, [&pb](TRuntimeNode /*n*/) { return pb.NewVoid(); }, false); + auto pgmReturn = pb.Contains(dict, items.front()); + + auto graph = setup.BuildGraph(pgmReturn); + auto res = graph->GetValue().template Get<bool>(); + UNIT_ASSERT_VALUES_EQUAL(res, true); + } + + Y_UNIT_TEST_LLVM(TestSqueezeToDict) { + auto test = [](bool stream, bool hashed, bool multi, bool compact, bool withPayload) { + Cerr << "TestSqueezeToDict [on: " << (stream ? "stream" : "flow") + << "type: " << (hashed ? "hashed" : "sorted") << ", multi: " << multi + << ", compact: " << compact << ", payload: " << withPayload << "]" << Endl; + + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + TVector<TRuntimeNode> items; + for (auto s : data) { + items.push_back(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(s)); + } + Shuffle(items.begin(), items.end()); + + auto dataType = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id); + auto list = pb.NewList(dataType, items); + auto input = stream ? pb.Iterator(list, items) : pb.ToFlow(list); + auto pgmReturn = hashed + ? pb.SqueezeToHashedDict(input, multi, [](TRuntimeNode n) { return n; }, + [&pb, withPayload](TRuntimeNode n) { return withPayload ? n : pb.NewVoid(); }, compact) + : pb.SqueezeToSortedDict(input, multi, [](TRuntimeNode n) { return n; }, + [&pb, withPayload](TRuntimeNode n) { return withPayload ? n : pb.NewVoid(); }, compact); + if (!stream) { + pgmReturn = pb.FromFlow(pgmReturn); + } + + auto graph = setup.BuildGraph(pgmReturn); + NUdf::TUnboxedValue res = graph->GetValue(); + UNIT_ASSERT(!res.IsSpecial()); + + NUdf::TUnboxedValue v; + auto status = res.Fetch(v); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, status); + + for (auto s : data) { + UNIT_ASSERT_C(v.Contains(NUdf::TUnboxedValue(MakeString(s))), s); + } + UNIT_ASSERT(!v.Contains(NUdf::TUnboxedValue(MakeString("green cucumber")))); + + status = res.Fetch(v); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, status); + }; + + for (auto stream : {true, false}) { + for (auto hashed : {true, false}) { + for (auto multi : {true, false}) { + for (auto compact : {true, false}) { + for (auto withPayload : {true, false}) { + test(stream, hashed, multi, compact, withPayload); + } + } + } + } + } + } +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 23u + Y_UNIT_TEST_LLVM(TestNarrowSqueezeToDict) { + auto test = [](bool hashed, bool multi, bool compact, bool withPayload) { + Cerr << "TestNarrowSqueezeToDict [type: " << (hashed ? "hashed" : "sorted") << ", multi: " << multi + << ", compact: " << compact << ", payload: " << withPayload << "]" << Endl; + + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + TVector<TRuntimeNode> items; + for (auto s : data) { + items.push_back(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(s)); + } + Shuffle(items.begin(), items.end()); + + auto dataType = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id); + auto list = pb.NewList(dataType, items); + auto input = pb.ExpandMap(pb.ToFlow(list), [](TRuntimeNode n) ->TRuntimeNode::TList { return {n}; }); + auto pgmReturn = hashed + ? pb.NarrowSqueezeToHashedDict(input, multi, [](TRuntimeNode::TList n) { return n.front(); }, + [&pb, withPayload](TRuntimeNode::TList n) { return withPayload ? n.back() : pb.NewVoid(); }, compact) + : pb.NarrowSqueezeToSortedDict(input, multi, [](TRuntimeNode::TList n) { return n.front(); }, + [&pb, withPayload](TRuntimeNode::TList n) { return withPayload ? n.back() : pb.NewVoid(); }, compact); + pgmReturn = pb.FromFlow(pgmReturn); + + auto graph = setup.BuildGraph(pgmReturn); + NUdf::TUnboxedValue res = graph->GetValue(); + UNIT_ASSERT(!res.IsSpecial()); + + NUdf::TUnboxedValue v; + auto status = res.Fetch(v); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, status); + + for (auto s : data) { + UNIT_ASSERT_C(v.Contains(NUdf::TUnboxedValue(MakeString(s))), s); + } + UNIT_ASSERT(!v.Contains(NUdf::TUnboxedValue(MakeString("green cucumber")))); + + status = res.Fetch(v); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, status); + }; + + for (auto hashed : {true, false}) { + for (auto multi : {true, false}) { + for (auto compact : {true, false}) { + for (auto withPayload : {true, false}) { + test(hashed, multi, compact, withPayload); + } + } + } + } + } +#endif + template <bool LLVM> + static void TestDictWithDataKeyImpl(bool optionalKey, bool multi, bool compact, bool withNull, bool withData) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + TType* keyType = pb.NewDataType(NUdf::EDataSlot::Int32, optionalKey); + TType* valueType = pb.NewDataType(NUdf::EDataSlot::Int32, false); + TType* tupleType = pb.NewTupleType({keyType, valueType}); + TVector<TRuntimeNode> items; + TVector<TRuntimeNode> keys; + if (withNull) { + UNIT_ASSERT(optionalKey); + keys.push_back(pb.NewEmptyOptional(keyType)); + for (size_t k = 0; k < 1 + multi; ++k) { + items.push_back(pb.NewTuple(tupleType, {keys.back(), pb.NewDataLiteral((i32)items.size())})); + } + } + if (withData) { + for (i32 i = 0; i < 2; ++i) { + auto key = pb.NewDataLiteral(i); + if (optionalKey) { + key = pb.NewOptional(key); + } + keys.push_back(key); + for (size_t k = 0; k < 1 + multi; ++k) { + items.push_back(pb.NewTuple(tupleType, {key, pb.NewDataLiteral((i32)items.size())})); + } + } + } + auto list = pb.NewList(tupleType, items); + auto keyList = pb.NewList(keyType, keys); + auto dict = pb.ToHashedDict(list, multi, [&](TRuntimeNode tuple) { return pb.Nth(tuple, 0); }, [&pb](TRuntimeNode tuple) { return pb.Nth(tuple, 1); }, compact); + + auto compareLists = [&](bool itemIsTuple, TRuntimeNode list1, TRuntimeNode list2) { + return pb.And({ + pb.Equals( + pb.Length(list1), + pb.Length(list2) + ), + pb.Not( + pb.Exists( + pb.Head( + pb.SkipWhile( + pb.Zip({list1, list2}), + [&](TRuntimeNode pair) { + if (itemIsTuple) { + return pb.And({ + pb.AggrEquals(pb.Nth(pb.Nth(pair, 0), 0), pb.Nth(pb.Nth(pair, 1), 0)), + pb.AggrEquals(pb.Nth(pb.Nth(pair, 0), 1), pb.Nth(pb.Nth(pair, 1), 1)), + }); + } else { + return pb.AggrEquals(pb.Nth(pair, 0), pb.Nth(pair, 1)); + } + } + ) + ) + ) + ) + }); + }; + + TVector<TRuntimeNode> results; + + // Check Dict has items + results.push_back(pb.AggrEquals( + pb.HasItems(dict), + pb.NewDataLiteral(withNull || withData) + )); + + // Check Dict length + results.push_back(pb.AggrEquals( + pb.Length(dict), + pb.NewDataLiteral((ui64)keys.size()) + )); + + // Check Dict Contains + results.push_back(pb.AllOf( + pb.Map(list, [&](TRuntimeNode tuple) { + return pb.Contains(dict, pb.Nth(tuple, 0)); + }), + [&](TRuntimeNode item) { return item; } + )); + + // Check Dict Lookup + results.push_back(compareLists(false, + pb.Sort( + pb.FlatMap( + pb.Map( + keyList, + [&](TRuntimeNode key) { + return pb.Unwrap(pb.Lookup(dict, key), pb.NewDataLiteral<NUdf::EDataSlot::String>("Lookup failed"), "", 0, 0); + } + ), + [&](TRuntimeNode item) { + return multi ? item : pb.NewOptional(item); + } + ), + pb.NewDataLiteral(true), + [&](TRuntimeNode item) { return item; } + ), + pb.Sort( + pb.Map(list, [&](TRuntimeNode tuple) { + return pb.Nth(tuple, 1); + }), + pb.NewDataLiteral(true), + [&](TRuntimeNode item) { return item; } + ) + )); + + // Check Dict items iterator + results.push_back(compareLists(true, + pb.Sort( + pb.FlatMap( + pb.DictItems(dict), + [&](TRuntimeNode pair) { + if (multi) { + return pb.Map( + pb.Nth(pair, 1), + [&](TRuntimeNode p) { + return pb.NewTuple({pb.Nth(pair, 0), p}); + } + ); + } else { + return pb.NewOptional(pair); + } + } + ), + pb.NewTuple({pb.NewDataLiteral(true), pb.NewDataLiteral(true)}), + [&](TRuntimeNode item) { return item; } + ), + list + )); + + // Check Dict payloads iterator + results.push_back(compareLists(false, + pb.Sort( + pb.FlatMap( + pb.DictPayloads(dict), + [&](TRuntimeNode item) { + return multi ? item : pb.NewOptional(item); + } + ), + pb.NewDataLiteral(true), + [&](TRuntimeNode item) { return item; } + ), + pb.Map( + list, + [&](TRuntimeNode item) { + return pb.Nth(item, 1); + } + ) + )); + + auto graph = setup.BuildGraph(pb.NewTuple(results)); + NUdf::TUnboxedValue res = graph->GetValue(); + + UNIT_ASSERT_C(res.GetElement(0).Get<bool>(), "Dict HasItems fail"); + UNIT_ASSERT_C(res.GetElement(1).Get<bool>(), "Dict Length fail"); + UNIT_ASSERT_C(res.GetElement(2).Get<bool>(), "Dict Contains fail"); + UNIT_ASSERT_C(res.GetElement(3).Get<bool>(), "Dict Lookup fail"); + UNIT_ASSERT_C(res.GetElement(4).Get<bool>(), "DictItems fail"); + UNIT_ASSERT_C(res.GetElement(5).Get<bool>(), "DictPayloads fail"); + } + + Y_UNIT_TEST_LLVM(TestDictWithDataKey) { + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*multi*/false, /*compact*/false, /*withNull*/false, /*withData*/true); + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*multi*/false, /*compact*/false, /*withNull*/false, /*withData*/false); // empty dict + } + + Y_UNIT_TEST_LLVM(TestDictCompactWithDataKey) { + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*multi*/false, /*compact*/true, /*withNull*/false, /*withData*/true); + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*multi*/false, /*compact*/true, /*withNull*/false, /*withData*/false); // empty dict + } + + Y_UNIT_TEST_LLVM(TestDictMultiWithDataKey) { + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*multi*/true, /*compact*/false, /*withNull*/false, /*withData*/true); + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*multi*/true, /*compact*/false, /*withNull*/false, /*withData*/false); // empty dict + } + + Y_UNIT_TEST_LLVM(TestDictCompactMultiWithDataKey) { + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*multi*/true, /*compact*/true, /*withNull*/false, /*withData*/true); + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*multi*/true, /*compact*/true, /*withNull*/false, /*withData*/false); // empty dict + } + + Y_UNIT_TEST_LLVM(TestDictWithOptionalDataKey) { + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/false, /*compact*/false, /*withNull*/false, /*withData*/true); + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/false, /*compact*/false, /*withNull*/true, /*withData*/false); + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/false, /*compact*/false, /*withNull*/true, /*withData*/true); + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/false, /*compact*/false, /*withNull*/false, /*withData*/false); // empty dict + } + + Y_UNIT_TEST_LLVM(TestDictCompactWithOptionalDataKey) { + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/false, /*compact*/true, /*withNull*/false, /*withData*/true); + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/false, /*compact*/true, /*withNull*/true, /*withData*/false); + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/false, /*compact*/true, /*withNull*/true, /*withData*/true); + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/false, /*compact*/true, /*withNull*/false, /*withData*/false); // empty dict + } + + Y_UNIT_TEST_LLVM(TestDictMultiWithOptionalDataKey) { + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/true, /*compact*/false, /*withNull*/false, /*withData*/true); + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/true, /*compact*/false, /*withNull*/true, /*withData*/false); + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/true, /*compact*/false, /*withNull*/true, /*withData*/true); + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/true, /*compact*/false, /*withNull*/false, /*withData*/false); // empty dict + } + + Y_UNIT_TEST_LLVM(TestDictCompactMultiWithOptionalDataKey) { + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/true, /*compact*/true, /*withNull*/false, /*withData*/true); + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/true, /*compact*/true, /*withNull*/true, /*withData*/false); + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/true, /*compact*/true, /*withNull*/true, /*withData*/true); + TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/true, /*compact*/true, /*withNull*/false, /*withData*/false); // empty dict + } + + template <bool LLVM> + static void TestSetWithDataKeyImpl(bool optionalKey, bool compact, bool withNull, bool withData) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + TType* keyType = pb.NewDataType(NUdf::EDataSlot::Int32, optionalKey); + TVector<TRuntimeNode> keys; + if (withNull) { + UNIT_ASSERT(optionalKey); + keys.push_back(pb.NewEmptyOptional(keyType)); + } + if (withData) { + for (i32 i = 0; i < 2; ++i) { + auto key = pb.NewDataLiteral(i); + if (optionalKey) { + key = pb.NewOptional(key); + } + keys.push_back(key); + } + } + auto keyList = pb.NewList(keyType, keys); + auto set = pb.ToHashedDict(keyList, false, [&](TRuntimeNode key) { return key; }, [&pb](TRuntimeNode) { return pb.NewVoid(); }, compact); + + auto compareLists = [&](TRuntimeNode list1, TRuntimeNode list2) { + return pb.And({ + pb.Equals( + pb.Length(list1), + pb.Length(list2) + ), + pb.Not( + pb.Exists( + pb.Head( + pb.SkipWhile( + pb.Zip({list1, list2}), + [&](TRuntimeNode pair) { + return pb.AggrEquals(pb.Nth(pair, 0), pb.Nth(pair, 1)); + } + ) + ) + ) + ) + }); + }; + + TVector<TRuntimeNode> results; + + // Check Set has items + results.push_back(pb.AggrEquals( + pb.HasItems(set), + pb.NewDataLiteral(withNull || withData) + )); + + // Check Set length + results.push_back(pb.AggrEquals( + pb.Length(set), + pb.NewDataLiteral((ui64)keys.size()) + )); + + // Check Set Contains + results.push_back(pb.AllOf( + pb.Map(keyList, [&](TRuntimeNode key) { + return pb.Contains(set, key); + }), + [&](TRuntimeNode item) { return item; } + )); + + // Check Set Lookup + results.push_back(pb.AllOf( + pb.Map(keyList, [&](TRuntimeNode key) { + return pb.Exists(pb.Lookup(set, key)); + }), + [&](TRuntimeNode item) { return item; } + )); + + // Check Set items iterator + results.push_back(compareLists( + pb.Sort( + pb.DictKeys(set), + pb.NewDataLiteral(true), + [&](TRuntimeNode item) { return item; } + ), + keyList + )); + + auto graph = setup.BuildGraph(pb.NewTuple(results)); + NUdf::TUnboxedValue res = graph->GetValue(); + + UNIT_ASSERT_C(res.GetElement(0).Get<bool>(), "Set HasItems fail"); + UNIT_ASSERT_C(res.GetElement(1).Get<bool>(), "Set Length fail"); + UNIT_ASSERT_C(res.GetElement(2).Get<bool>(), "Set Contains fail"); + UNIT_ASSERT_C(res.GetElement(3).Get<bool>(), "Set Lookup fail"); + UNIT_ASSERT_C(res.GetElement(4).Get<bool>(), "Set DictKeys fail"); + } + + Y_UNIT_TEST_LLVM(TestSetWithDataKey) { + TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*compact*/false, /*withNull*/false, /*withData*/true); + TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*compact*/false, /*withNull*/false, /*withData*/false); // empty set + } + + Y_UNIT_TEST_LLVM(TestSetCompactWithDataKey) { + TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*compact*/true, /*withNull*/false, /*withData*/true); + TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*compact*/true, /*withNull*/false, /*withData*/false); // empty set + } + + Y_UNIT_TEST_LLVM(TestSetWithOptionalDataKey) { + TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*compact*/false, /*withNull*/false, /*withData*/true); + TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*compact*/false, /*withNull*/true, /*withData*/false); + TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*compact*/false, /*withNull*/true, /*withData*/true); + TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*compact*/false, /*withNull*/false, /*withData*/false); // empty set + } + + Y_UNIT_TEST_LLVM(TestSetCompactWithOptionalDataKey) { + TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*compact*/true, /*withNull*/false, /*withData*/true); + TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*compact*/true, /*withNull*/true, /*withData*/false); + TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*compact*/true, /*withNull*/true, /*withData*/true); + TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*compact*/true, /*withNull*/false, /*withData*/false); // empty set + } +} + + +} // namespace diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_variant_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_variant_ut.cpp new file mode 100644 index 00000000000..2a52f73ef33 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_variant_ut.cpp @@ -0,0 +1,527 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> + +namespace NKikimr { +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TMiniKQLVariantTest) { + Y_UNIT_TEST_LLVM(TestGuessTuple) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc"); + const auto tupleType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id)}); + const auto varType = pb.NewVariantType(tupleType); + const auto var1 = pb.NewVariant(data1, 0, varType); + const auto var2 = pb.NewVariant(data2, 1, varType); + std::vector<TRuntimeNode> tupleItems; + tupleItems.push_back(pb.Guess(var1, 0)); + tupleItems.push_back(pb.Guess(var1, 1)); + tupleItems.push_back(pb.Guess(var2, 0)); + tupleItems.push_back(pb.Guess(var2, 1)); + const auto pgmReturn = pb.NewTuple(tupleItems); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto res = graph->GetValue(); + UNIT_ASSERT(res.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(res.GetElement(0).template Get<ui32>(), 1); + UNIT_ASSERT(!res.GetElement(1)); + UNIT_ASSERT(!res.GetElement(2)); + UNIT_ASSERT(res.GetElement(3)); + UNBOXED_VALUE_STR_EQUAL(res.GetElement(3), "abc"); + } + + Y_UNIT_TEST_LLVM(TestGuessTupleOpt) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc"); + const auto tupleType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id)}); + const auto varType = pb.NewVariantType(tupleType); + const auto var1 = pb.NewVariant(data1, 0, varType); + const auto var2 = pb.NewVariant(data2, 1, varType); + const auto jvar1 = pb.NewOptional(var1); + const auto jvar2 = pb.NewOptional(var2); + const auto nothing = pb.NewEmptyOptional(pb.NewOptionalType(varType)); + + std::vector<TRuntimeNode> tupleItems; + tupleItems.push_back(pb.Guess(jvar1, 0)); + tupleItems.push_back(pb.Guess(jvar1, 1)); + tupleItems.push_back(pb.Guess(jvar2, 0)); + tupleItems.push_back(pb.Guess(jvar2, 1)); + tupleItems.push_back(pb.Guess(nothing, 0)); + tupleItems.push_back(pb.Guess(nothing, 1)); + const auto pgmReturn = pb.NewTuple(tupleItems); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto res = graph->GetValue(); + UNIT_ASSERT(res.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(res.GetElement(0).template Get<ui32>(), 1); + UNIT_ASSERT(!res.GetElement(1)); + UNIT_ASSERT(!res.GetElement(2)); + UNIT_ASSERT(res.GetElement(3)); + UNBOXED_VALUE_STR_EQUAL(res.GetElement(3), "abc"); + UNIT_ASSERT(!res.GetElement(4)); + UNIT_ASSERT(!res.GetElement(5)); + } + + Y_UNIT_TEST_LLVM(TestGuessStruct) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc"); + const auto structType = pb.NewStructType({{"x", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, {"y", pb.NewDataType(NUdf::TDataType<char*>::Id)}}); + const auto varType = pb.NewVariantType(structType); + const auto var1 = pb.NewVariant(data1, "x", varType); + const auto var2 = pb.NewVariant(data2, "y", varType); + std::vector<TRuntimeNode> tupleItems; + tupleItems.push_back(pb.Guess(var1, "x")); + tupleItems.push_back(pb.Guess(var1, "y")); + tupleItems.push_back(pb.Guess(var2, "x")); + tupleItems.push_back(pb.Guess(var2, "y")); + const auto pgmReturn = pb.NewTuple(tupleItems); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto res = graph->GetValue(); + UNIT_ASSERT(res.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(res.GetElement(0).template Get<ui32>(), 1); + UNIT_ASSERT(!res.GetElement(1)); + UNIT_ASSERT(!res.GetElement(2)); + UNIT_ASSERT(res.GetElement(3)); + UNBOXED_VALUE_STR_EQUAL(res.GetElement(3), "abc"); + } + + Y_UNIT_TEST_LLVM(TestGuessStructOpt) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc"); + const auto structType = pb.NewStructType({{"x", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, {"y", pb.NewDataType(NUdf::TDataType<char*>::Id)}}); + const auto varType = pb.NewVariantType(structType); + const auto var1 = pb.NewVariant(data1, "x", varType); + const auto var2 = pb.NewVariant(data2, "y", varType); + const auto jvar1 = pb.NewOptional(var1); + const auto jvar2 = pb.NewOptional(var2); + const auto nothing = pb.NewEmptyOptional(pb.NewOptionalType(varType)); + + std::vector<TRuntimeNode> tupleItems; + tupleItems.push_back(pb.Guess(jvar1, "x")); + tupleItems.push_back(pb.Guess(jvar1, "y")); + tupleItems.push_back(pb.Guess(jvar2, "x")); + tupleItems.push_back(pb.Guess(jvar2, "y")); + tupleItems.push_back(pb.Guess(nothing, "x")); + tupleItems.push_back(pb.Guess(nothing, "y")); + const auto pgmReturn = pb.NewTuple(tupleItems); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto res = graph->GetValue(); + UNIT_ASSERT(res.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(res.GetElement(0).template Get<ui32>(), 1); + UNIT_ASSERT(!res.GetElement(1)); + UNIT_ASSERT(!res.GetElement(2)); + UNIT_ASSERT(res.GetElement(3)); + UNBOXED_VALUE_STR_EQUAL(res.GetElement(3), "abc"); + UNIT_ASSERT(!res.GetElement(4)); + UNIT_ASSERT(!res.GetElement(5)); + } + + Y_UNIT_TEST_LLVM(TestVisitAllTuple) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc"); + const auto at = pb.NewDataLiteral<NUdf::EDataSlot::String>("@"); + const auto tupleType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id)}); + const auto varType = pb.NewVariantType(tupleType); + const auto var1 = pb.NewVariant(data1, 0, varType); + const auto var2 = pb.NewVariant(data2, 1, varType); + const auto list = pb.NewList(varType, {var1, var2}); + const auto pgmReturn = pb.Map(list, [&](TRuntimeNode item) { + return pb.VisitAll(item, [&](ui32 index, TRuntimeNode item) { + if (!index) { + return pb.Concat(at, pb.ToString(item)); + } else { + return pb.Concat(at, item); + } + }); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto res = graph->GetValue(); + UNBOXED_VALUE_STR_EQUAL(res.GetElement(0), "@1"); + UNBOXED_VALUE_STR_EQUAL(res.GetElement(1), "@abc"); + } + + Y_UNIT_TEST_LLVM(TestVisitAllStruct) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto at = pb.NewDataLiteral<NUdf::EDataSlot::String>("@"); + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc"); + const auto structType = pb.NewStructType({{"x", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, {"y", pb.NewDataType(NUdf::TDataType<char*>::Id)}}); + const auto varType = pb.NewVariantType(structType); + const auto var1 = pb.NewVariant(data1, "x", varType); + const auto var2 = pb.NewVariant(data2, "y", varType); + const auto list = pb.NewList(varType, {var1, var2}); + + const auto xIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("x"); + const auto pgmReturn = pb.Map(list, [&](TRuntimeNode item) { + return pb.VisitAll(item, [&](ui32 index, TRuntimeNode item) { + if (xIndex == index) { + return pb.Concat(at, pb.ToString(item)); + } else { + return pb.Concat(at, item); + } + }); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto res = graph->GetValue(); + UNBOXED_VALUE_STR_EQUAL(res.GetElement(0), "@1"); + UNBOXED_VALUE_STR_EQUAL(res.GetElement(1), "@abc"); + } + + Y_UNIT_TEST_LLVM(TestVisitAllTupleFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto at = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("@"); + const auto data1 = pb.NewDataLiteral<ui64>(3ULL); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("abc"); + const auto tupleType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui64>::Id), pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)}); + const auto varType = pb.NewVariantType(tupleType); + const auto var1 = pb.NewVariant(data1, 0, varType); + const auto var2 = pb.NewVariant(data2, 1, varType); + const auto list = pb.NewList(varType, {var1, var2}); + const auto pgmReturn = pb.FromFlow(pb.FlatMap(pb.ToFlow(list), [&](TRuntimeNode item) { + return pb.VisitAll(item, [&](ui32 index, TRuntimeNode item) { + if (!index) { + return pb.ToFlow(pb.Replicate(at, item, __FILE__, __LINE__, 0U)); + } else { + return pb.ToFlow(pb.NewOptional(item)); + } + }); + })); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto res = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok); + UNBOXED_VALUE_STR_EQUAL(item, "@"); + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok); + UNBOXED_VALUE_STR_EQUAL(item, "@"); + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok); + UNBOXED_VALUE_STR_EQUAL(item, "@"); + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok); + UNBOXED_VALUE_STR_EQUAL(item, "abc"); + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Finish); + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Finish); + } + + Y_UNIT_TEST_LLVM(TestVisitAllStructFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto at = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("@"); + const auto data1 = pb.NewDataLiteral<ui64>(4ULL); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("abc"); + const auto structType = pb.NewStructType({{"x", pb.NewDataType(NUdf::TDataType<ui64>::Id)}, {"y", pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)}}); + const auto varType = pb.NewVariantType(structType); + const auto var1 = pb.NewVariant(data1, "x", varType); + const auto var2 = pb.NewVariant(data2, "y", varType); + const auto list = pb.NewList(varType, {var2, var1, var2}); + + const auto xIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("x"); + const auto pgmReturn = pb.FromFlow(pb.FlatMap(pb.ToFlow(list), [&](TRuntimeNode item) { + return pb.VisitAll(item, [&](ui32 index, TRuntimeNode item) { + if (xIndex == index) { + return pb.ToFlow(pb.Replicate(at, item, __FILE__, __LINE__, 0U)); + } else { + return pb.ToFlow(pb.NewOptional(item)); + } + }); + })); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto res = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok); + UNBOXED_VALUE_STR_EQUAL(item, "abc"); + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok); + UNBOXED_VALUE_STR_EQUAL(item, "@"); + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok); + UNBOXED_VALUE_STR_EQUAL(item, "@"); + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok); + UNBOXED_VALUE_STR_EQUAL(item, "@"); + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok); + UNBOXED_VALUE_STR_EQUAL(item, "@"); + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok); + UNBOXED_VALUE_STR_EQUAL(item, "abc"); + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Finish); + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Finish); + } + + Y_UNIT_TEST_LLVM(TestVisitAllStructWideFlow) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto at = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("@"); + const auto data1 = pb.NewDataLiteral<ui64>(4ULL); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("abc"); + const auto structType = pb.NewStructType({{"x", pb.NewDataType(NUdf::TDataType<ui64>::Id)}, {"y", pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)}}); + const auto varType = pb.NewVariantType(structType); + const auto var1 = pb.NewVariant(data1, "x", varType); + const auto var2 = pb.NewVariant(data2, "y", varType); + const auto list = pb.NewList(varType, {var2, var1, var2}); + + const auto xIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("x"); + const auto pgmReturn = pb.FromFlow(pb.NarrowMap(pb.FlatMap(pb.ToFlow(list), [&](TRuntimeNode item) { + return pb.VisitAll(item, [&](ui32 index, TRuntimeNode item) { + if (xIndex == index) { + return pb.ExpandMap(pb.ToFlow(pb.Replicate(at, item, __FILE__, __LINE__, 0U)), [&](TRuntimeNode x) -> TRuntimeNode::TList { return {x, pb.NewDataLiteral(i32(index))}; }); + } else { + return pb.ExpandMap(pb.ToFlow(pb.NewOptional(item)), [&](TRuntimeNode x) -> TRuntimeNode::TList { return {x, pb.NewDataLiteral(-i32(index))}; }); + } + }); + }), + [&](TRuntimeNode::TList items) { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto res = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "abc"); + UNIT_ASSERT_EQUAL(item.GetElement(1).Get<i32>(), -1); + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "@"); + UNIT_ASSERT_EQUAL(item.GetElement(1).Get<i32>(), 0); + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "@"); + UNIT_ASSERT_EQUAL(item.GetElement(1).Get<i32>(), 0); + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "@"); + UNIT_ASSERT_EQUAL(item.GetElement(1).Get<i32>(), 0); + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "@"); + UNIT_ASSERT_EQUAL(item.GetElement(1).Get<i32>(), 0); + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "abc"); + UNIT_ASSERT_EQUAL(item.GetElement(1).Get<i32>(), -1); + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Finish); + UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Finish); + } + + Y_UNIT_TEST_LLVM(TestWayTuple) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc"); + const auto tupleType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id)}); + const auto varType = pb.NewVariantType(tupleType); + const auto var1 = pb.NewVariant(data1, 0, varType); + const auto var2 = pb.NewVariant(data2, 1, varType); + const auto list = pb.NewList(varType, {var2, var1}); + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { return pb.Way(item); } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestWayTupleOpt) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc"); + const auto tupleType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id)}); + const auto varType = pb.NewVariantType(tupleType); + const auto var1 = pb.NewVariant(data1, 0, varType); + const auto var2 = pb.NewVariant(data2, 1, varType); + const auto jvar1 = pb.NewOptional(var1); + const auto jvar2 = pb.NewOptional(var2); + const auto optType = pb.NewOptionalType(varType); + const auto nothing = pb.NewEmptyOptional(optType); + const auto list = pb.NewList(optType, {jvar2, nothing, jvar1}); + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { return pb.Way(item); } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1U); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestWayStruct) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc"); + const auto structType = pb.NewStructType({{"x", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, {"y", pb.NewDataType(NUdf::TDataType<char*>::Id)}}); + const auto varType = pb.NewVariantType(structType); + const auto var1 = pb.NewVariant(data1, "x", varType); + const auto var2 = pb.NewVariant(data2, "y", varType); + const auto list = pb.NewList(varType, {var2, var1}); + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { return pb.Way(item); } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "y"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "x"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestWayStructOpt) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data1 = pb.NewDataLiteral<ui32>(1); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc"); + const auto structType = pb.NewStructType({{"x", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, {"y", pb.NewDataType(NUdf::TDataType<char*>::Id)}}); + const auto varType = pb.NewVariantType(structType); + const auto var1 = pb.NewVariant(data1, "x", varType); + const auto var2 = pb.NewVariant(data2, "y", varType); + const auto jvar1 = pb.NewOptional(var1); + const auto jvar2 = pb.NewOptional(var2); + const auto optType = pb.NewOptionalType(varType); + const auto nothing = pb.NewEmptyOptional(optType); + const auto list = pb.NewList(optType, {jvar2, nothing, jvar1}); + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { return pb.Way(item); } + ); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "y"); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "x"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestItemInMap) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto varType = pb.NewVariantType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id), pb.NewDataType(NUdf::TDataType<bool>::Id)})); + + const auto data0 = pb.NewVariant(pb.NewDataLiteral<i32>(77), 0, varType); + const auto data1 = pb.NewVariant(pb.NewDataLiteral<NUdf::EDataSlot::String>("abc"), 1, varType); + const auto data2 = pb.NewVariant(pb.NewDataLiteral<bool>(false), 2, varType); + const auto data3 = pb.NewVariant(pb.NewDataLiteral<bool>(true), 2, varType); + const auto data4 = pb.NewVariant(pb.NewDataLiteral<NUdf::EDataSlot::String>("DEF"), 1, varType); + const auto data5 = pb.NewVariant(pb.NewDataLiteral<i32>(-1267), 0, varType); + const auto list = pb.NewList(varType, {data0, data1, data2, data3, data4, data5}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.VariantItem(item); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 77); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(item.AsStringRef()), "abc"); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(item.AsStringRef()), "DEF"); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1267); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestGuessInMap) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto varType = pb.NewVariantType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id), pb.NewDataType(NUdf::TDataType<bool>::Id)})); + + const auto data0 = pb.NewVariant(pb.NewDataLiteral<i32>(77), 0, varType); + const auto data1 = pb.NewVariant(pb.NewDataLiteral<NUdf::EDataSlot::String>("abc"), 1, varType); + const auto data2 = pb.NewVariant(pb.NewDataLiteral<bool>(false), 2, varType); + const auto data3 = pb.NewVariant(pb.NewDataLiteral<bool>(true), 2, varType); + const auto data4 = pb.NewVariant(pb.NewDataLiteral<NUdf::EDataSlot::String>("DEF"), 1, varType); + const auto data5 = pb.NewVariant(pb.NewDataLiteral<i32>(-1267), 0, varType); + const auto list = pb.NewList(varType, {data0, data1, data2, data3, data4, data5}); + + const auto pgmReturn = pb.Map(list, + [&](TRuntimeNode item) { + return pb.NewTuple({pb.Guess(item, 0), pb.Guess(item, 2)}); + }); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 77); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<bool>(), false); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<bool>(), true); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -1267); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chain_map_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chain_map_ut.cpp new file mode 100644 index 00000000000..e5ac540ca0e --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chain_map_ut.cpp @@ -0,0 +1,316 @@ +#include "mkql_computation_node_ut.h" +#include <ydb/library/yql/minikql/mkql_runtime_version.h> + +namespace NKikimr { +namespace NMiniKQL { +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 23u +Y_UNIT_TEST_SUITE(TMiniKQLWideChain1MapTest) { + Y_UNIT_TEST_LLVM(TestThinLambda) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType)}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4))}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChain1Map(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode) -> TRuntimeNode::TList { return {}; }), + [&](TRuntimeNode::TList inputs) { return inputs; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList outputs) { return outputs; }), + [&](TRuntimeNode::TList) -> TRuntimeNode { return pb.NewTuple({}); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestSimpleSwap) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChain1Map(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList inputs) { return inputs; }, + [&](TRuntimeNode::TList inputs, TRuntimeNode::TList outputs) -> TRuntimeNode::TList { return {inputs.back(), outputs[1U], inputs.front()}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -2); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(!item.GetElement(2)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -3); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 4); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestSimpleChain) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewOptional(pb.NewDataLiteral<i32>(-5)), pb.NewOptional(pb.NewDataLiteral<i32>(-6))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-4)), pb.NewOptional(pb.NewDataLiteral<i32>(-7))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewOptional(pb.NewDataLiteral<i32>(-7)), pb.NewOptional(pb.NewDataLiteral<i32>(-8))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-9))}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.FromFlow(pb.NarrowMap(pb.WideChain1Map(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList inputs) -> TRuntimeNode::TList { return {pb.Add(inputs.front(), inputs[1U]), pb.NewEmptyOptional(dataType), pb.Sub(inputs.back(), inputs[1U])}; }, + [&](TRuntimeNode::TList inputs, TRuntimeNode::TList outputs) -> TRuntimeNode::TList { + return {pb.AggrAdd(outputs.back(), inputs[1U]), outputs.front(), pb.Decrement(outputs[1])}; + }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(NUdf::EFetchStatus::Ok == iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -4); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -1); + UNIT_ASSERT(NUdf::EFetchStatus::Ok == iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -5); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -4); + UNIT_ASSERT(!item.GetElement(2)); + UNIT_ASSERT(NUdf::EFetchStatus::Ok == iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -7); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -5); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -5); + UNIT_ASSERT(NUdf::EFetchStatus::Ok == iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -5); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -7); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -6); + UNIT_ASSERT(NUdf::EFetchStatus::Finish == iterator.Fetch(item)); + UNIT_ASSERT(NUdf::EFetchStatus::Finish == iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestAgrregateWithPrevious) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(5)), pb.NewEmptyOptional(dataType)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChain1Map(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList inputs) -> TRuntimeNode::TList { return {inputs[1U], inputs[2U], inputs[0U]}; }, + [&](TRuntimeNode::TList inputs, TRuntimeNode::TList outputs) -> TRuntimeNode::TList { + return {pb.AggrMin(inputs[0U], outputs[1U]), pb.AggrMax(inputs[1U], outputs[2U]), pb.AggrAdd(outputs[0U], inputs[2U])}; + }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 2); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -2); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -2); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 5); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 2); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestPasstroughtFieldSplitAsIs) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewOptional(pb.NewDataLiteral<i32>(-6)), pb.NewOptional(pb.NewDataLiteral<i32>(-5))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-4)), pb.NewOptional(pb.NewDataLiteral<i32>(-4))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewOptional(pb.NewDataLiteral<i32>(-7)), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(0))}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChain1Map(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList inputs) -> TRuntimeNode::TList { return {inputs[1U], pb.Mul(inputs.front(), inputs.back()), inputs[1U]}; }, + [&](TRuntimeNode::TList inputs, TRuntimeNode::TList outputs) -> TRuntimeNode::TList { + return {inputs[1U], pb.Mul(outputs[1U], pb.Add(inputs.back(), inputs.front())), inputs[1U]}; + }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -6); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -5); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -6); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -4); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 10); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -7); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -7); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 0); + UNIT_ASSERT(!item.GetElement(2)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFieldBothWayPasstroughtAndArg) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewOptional(pb.NewDataLiteral<i32>(-5)), pb.NewOptional(pb.NewDataLiteral<i32>(-6))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-4)), pb.NewOptional(pb.NewDataLiteral<i32>(-7))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewOptional(pb.NewDataLiteral<i32>(-7)), pb.NewOptional(pb.NewDataLiteral<i32>(-8))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-9))}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChain1Map(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList inputs) -> TRuntimeNode::TList { return {inputs[1U], pb.Sub(inputs.front(), inputs.back()), pb.Minus(inputs[1U])}; }, + [&](TRuntimeNode::TList inputs, TRuntimeNode::TList outputs) -> TRuntimeNode::TList { + return {inputs[1U], pb.Sub(outputs[1U], pb.Add(inputs.back(), inputs.front())), pb.Minus(inputs[1U])}; + }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -5); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 7); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 5); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -4); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 12); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -7); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 17); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 7); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 22); + UNIT_ASSERT(!item.GetElement(2)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestDotCalculateUnusedField) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + const auto null = pb.NewEmptyOptional(dataType); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), null, pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), null, pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), null, pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), null, pb.NewOptional(pb.NewDataLiteral<i32>(-4))}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("Veszély! Aknák!"); + + const auto pgmReturn = pb.FromFlow(pb.NarrowMap(pb.WideChain1Map(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList inputs) -> TRuntimeNode::TList { return {inputs.back(), pb.Unwrap(inputs[1U], landmine, __FILE__, __LINE__, 0), inputs.front()}; }, + [&](TRuntimeNode::TList inputs, TRuntimeNode::TList outputs) -> TRuntimeNode::TList { + return {pb.Mul(outputs.front(), inputs.back()), pb.Unwrap(inputs[1U], landmine, __FILE__, __LINE__, 0), pb.Add(inputs.front(), outputs.back())}; + }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple({items.back(), items.front()}); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(NUdf::EFetchStatus::Ok == iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -1); + UNIT_ASSERT(NUdf::EFetchStatus::Ok == iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 3); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2); + UNIT_ASSERT(NUdf::EFetchStatus::Ok == iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 6); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -6); + UNIT_ASSERT(NUdf::EFetchStatus::Ok == iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 10); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 24); + UNIT_ASSERT(NUdf::EFetchStatus::Finish == iterator.Fetch(item)); + UNIT_ASSERT(NUdf::EFetchStatus::Finish == iterator.Fetch(item)); + } +} +#endif +} +} + diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chopper_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chopper_ut.cpp new file mode 100644 index 00000000000..b2865328e86 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chopper_ut.cpp @@ -0,0 +1,474 @@ +#include "mkql_computation_node_ut.h" +#include <ydb/library/yql/minikql/mkql_runtime_version.h> + +namespace NKikimr { +namespace NMiniKQL { +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u +Y_UNIT_TEST_SUITE(TMiniKQLWideChopperTest) { + Y_UNIT_TEST_LLVM(TestConcatKeyToItems) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto tupleType = pb.NewTupleType({dataType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one"); + const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9"); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, value1}); + + const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3}); + + const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4}); + + const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5}); + const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6}); + const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7}); + const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8}); + const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChopper(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { + return {pb.Substring(items.front(), pb.Sub(pb.Size(items.front()), pb.NewDataLiteral<ui32>(4U)), pb.NewDataLiteral<ui32>(4U))}; + }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) { + return pb.AggrNotEquals(keys.front(), items.front()); + }, + [&](TRuntimeNode::TList keys, TRuntimeNode input) { + return pb.WideMap(input, [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { + return {pb.AggrConcat(items.back(), keys.front())}; + }); + }), + [&](TRuntimeNode::TList items) { return items.front(); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 1 one"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 2 two"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 3 two"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 4 one"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 5 two"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 6 two"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 7 two"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 8 two"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 9 two"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestCollectKeysOnly) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto tupleType = pb.NewTupleType({dataType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one"); + const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 9"); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, value1}); + + const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3}); + + const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4}); + + const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5}); + const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6}); + const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7}); + const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8}); + const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChopper(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { + return {items.front()}; + }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) { + return pb.AggrNotEquals(keys.front(), items.front()); + }, + [&](TRuntimeNode::TList keys, TRuntimeNode) { + return pb.ExpandMap(pb.ToFlow(pb.NewOptional(keys.front())), [&](TRuntimeNode item) -> TRuntimeNode::TList { + return {item}; + } ); + }), + [&](TRuntimeNode::TList items) { return items.front(); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "key one"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "key two"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long key one"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long key two"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestGetPart) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto tupleType = pb.NewTupleType({dataType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one"); + const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 9"); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, value1}); + + const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3}); + + const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4}); + const auto data5 = pb.NewTuple(tupleType, {longKeyOne, value5}); + const auto data6 = pb.NewTuple(tupleType, {longKeyOne, value6}); + const auto data7 = pb.NewTuple(tupleType, {longKeyOne, value7}); + const auto data8 = pb.NewTuple(tupleType, {longKeyOne, value8}); + + const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChopper(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { + return {items.front()}; + }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) { + return pb.AggrNotEquals(keys.front(), items.front()); + }, + [&](TRuntimeNode::TList, TRuntimeNode input) { + return pb.Take(pb.Skip(input, pb.NewDataLiteral<ui64>(1ULL)), pb.NewDataLiteral<ui64>(3ULL)); + }), + [&](TRuntimeNode::TList items) { return items.back(); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 3"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 5"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 6"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 7"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestSwitchByBoolFieldAndDontUseKey) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto boolType = pb.NewDataType(NUdf::TDataType<bool>::Id); + const auto tupleType = pb.NewTupleType({pb.NewOptionalType(dataType), dataType, boolType}); + + const auto key0 = pb.NewEmptyOptional(pb.NewOptionalType(dataType)); + const auto key1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("one")); + const auto key2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("two")); + + const auto trueVal = pb.NewDataLiteral<bool>(true); + const auto falseVal = pb.NewDataLiteral<bool>(false); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 9"); + + const auto data1 = pb.NewTuple(tupleType, {key0, value1, trueVal}); + const auto data2 = pb.NewTuple(tupleType, {key1, value2, falseVal}); + const auto data3 = pb.NewTuple(tupleType, {key2, value3, falseVal}); + const auto data4 = pb.NewTuple(tupleType, {key0, value4, trueVal}); + const auto data5 = pb.NewTuple(tupleType, {key1, value5, falseVal}); + const auto data6 = pb.NewTuple(tupleType, {key2, value6, falseVal}); + const auto data7 = pb.NewTuple(tupleType, {key0, value7, falseVal}); + const auto data8 = pb.NewTuple(tupleType, {key1, value8, falseVal}); + const auto data9 = pb.NewTuple(tupleType, {key2, value9, trueVal}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::String>("ACHTUNG MINEN!"); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChopper(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { + return {pb.Unwrap(items.front(), landmine, __FILE__, __LINE__, 0)}; + }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items) { + return items.back(); + }, + [&](TRuntimeNode::TList, TRuntimeNode input) { + return pb.Take(input, pb.NewDataLiteral<ui64>(2ULL)); + }), + [&](TRuntimeNode::TList items) { return items[1U]; } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 1"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 2"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 4"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 5"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 9"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestCollectKeysIfPresent) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto boolType = pb.NewDataType(NUdf::TDataType<bool>::Id); + const auto tupleType = pb.NewTupleType({pb.NewOptionalType(dataType), dataType, boolType}); + + const auto key0 = pb.NewEmptyOptional(pb.NewOptionalType(dataType)); + const auto key1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("one")); + const auto key2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("two")); + + const auto trueVal = pb.NewDataLiteral<bool>(true); + const auto falseVal = pb.NewDataLiteral<bool>(false); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 9"); + + const auto data1 = pb.NewTuple(tupleType, {key1, value1, trueVal}); + const auto data2 = pb.NewTuple(tupleType, {key1, value2, falseVal}); + const auto data3 = pb.NewTuple(tupleType, {key1, value3, falseVal}); + const auto data4 = pb.NewTuple(tupleType, {key0, value4, trueVal}); + const auto data5 = pb.NewTuple(tupleType, {key0, value5, falseVal}); + const auto data6 = pb.NewTuple(tupleType, {key2, value6, falseVal}); + const auto data7 = pb.NewTuple(tupleType, {key0, value7, falseVal}); + const auto data8 = pb.NewTuple(tupleType, {key0, value8, falseVal}); + const auto data9 = pb.NewTuple(tupleType, {key0, value9, trueVal}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChopper(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { + return {items.front()}; + }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) { + return pb.AggrNotEquals(keys.front(), items.front()); + }, + [&](TRuntimeNode::TList keys, TRuntimeNode part) { + return pb.IfPresent(keys, + [&](TRuntimeNode::TList keys) { return pb.ExpandMap(pb.ToFlow(pb.NewList(dataType, keys)), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; } ); }, + pb.WideMap(part, [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items[1U]}; } )); + }), + [&](TRuntimeNode::TList items) { return items.front(); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "one"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 4"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 5"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "two"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 7"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 8"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 9"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestConditionalByKeyPart) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto boolType = pb.NewDataType(NUdf::TDataType<bool>::Id); + const auto tupleType = pb.NewTupleType({pb.NewOptionalType(dataType), dataType, boolType}); + + const auto key0 = pb.NewEmptyOptional(pb.NewOptionalType(dataType)); + const auto key1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("one")); + const auto key2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("two")); + + const auto trueVal = pb.NewDataLiteral<bool>(true); + const auto falseVal = pb.NewDataLiteral<bool>(false); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 9"); + + const auto data1 = pb.NewTuple(tupleType, {key1, value1, trueVal}); + const auto data2 = pb.NewTuple(tupleType, {key1, value2, trueVal}); + const auto data3 = pb.NewTuple(tupleType, {key1, value3, falseVal}); + const auto data4 = pb.NewTuple(tupleType, {key1, value4, falseVal}); + const auto data5 = pb.NewTuple(tupleType, {key2, value5, falseVal}); + const auto data6 = pb.NewTuple(tupleType, {key2, value6, falseVal}); + const auto data7 = pb.NewTuple(tupleType, {key2, value7, trueVal}); + const auto data8 = pb.NewTuple(tupleType, {key0, value8, trueVal}); + const auto data9 = pb.NewTuple(tupleType, {key0, value9, falseVal}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChopper(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { + return {items.front(), items.back()}; + }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) { + return pb.Or({pb.AggrNotEquals(keys.front(), items.front()), pb.AggrNotEquals(keys.back(), items.back())}); + }, + [&](TRuntimeNode::TList keys, TRuntimeNode part) { + return pb.If(keys.back(), + pb.ExpandMap(pb.ToFlow(keys.front()), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; } ), + pb.WideMap(part, [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items[1U]}; } )); + }), + [&](TRuntimeNode::TList items) { return items.front(); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "one"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 3"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 4"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 5"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 6"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "two"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 9"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestThinAllLambdas) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto tupleType = pb.NewTupleType({}); + + + const auto data = pb.NewTuple({}); + + const auto list = pb.NewList(tupleType, {data, data, data, data}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChopper(pb.ExpandMap(pb.ToFlow(list), + [](TRuntimeNode) -> TRuntimeNode::TList { return {}; }), + [](TRuntimeNode::TList items) { return items; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList) { return pb.NewDataLiteral<bool>(true); }, + [&](TRuntimeNode::TList, TRuntimeNode input) { return pb.WideMap(input, [](TRuntimeNode::TList items) { return items; }); }), + [&](TRuntimeNode::TList) { return pb.NewTuple({}); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } +} +#endif +} +} + diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp new file mode 100644 index 00000000000..6d8a18901b1 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp @@ -0,0 +1,1699 @@ +#include "mkql_computation_node_ut.h" +#include <ydb/library/yql/minikql/mkql_runtime_version.h> + +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> + +#include <cstring> +#include <random> +#include <algorithm> + +namespace NKikimr { +namespace NMiniKQL { +namespace { +const auto border = 9124596000000000ULL; +} + +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u +Y_UNIT_TEST_SUITE(TMiniKQLWideCombinerTest) { + Y_UNIT_TEST_LLVM(TestLongStringsRefCounting) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto optionalType = pb.NewOptionalType(dataType); + const auto tupleType = pb.NewTupleType({dataType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one"); + const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9"); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, value1}); + + const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3}); + + const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4}); + + const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5}); + const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6}); + const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7}); + const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8}); + const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), 0ULL, + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) -> TRuntimeNode::TList { + return {pb.NewOptional(items.back()), pb.NewOptional(keys.front()), pb.NewEmptyOptional(optionalType), pb.NewEmptyOptional(optionalType)}; + }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { + return {pb.NewOptional(items.back()), state.front(), state[1U], state[2U]}; + }, + [&](TRuntimeNode::TList, TRuntimeNode::TList state) -> TRuntimeNode::TList { + state.erase(state.cbegin()); + return {pb.FlatMap(pb.NewList(optionalType, state), [&](TRuntimeNode item) { return item; } )}; + }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { + return pb.Fold1(items.front(), + [&](TRuntimeNode item) { return item; }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.AggrConcat(pb.AggrConcat(state, pb.NewDataLiteral<NUdf::EDataSlot::String>(" / ")), item); + } + ); + } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "key one"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 2 / key two"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long key one"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 8 / very long value 7 / very long value 6"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestLongStringsPasstroughtRefCounting) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto tupleType = pb.NewTupleType({dataType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one"); + const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9"); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, value1}); + + const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3}); + + const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4}); + + const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5}); + const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6}); + const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7}); + const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8}); + const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), 0ULL, + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) -> TRuntimeNode::TList { + return {items.back(), keys.front(), items.back(), items.front()}; + }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { + return {items.back(), keys.front(), state[2U], state.back()}; + }, + [&](TRuntimeNode::TList, TRuntimeNode::TList state) -> TRuntimeNode::TList { + return state; + }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { + return pb.Fold1(pb.NewList(dataType, items), + [&](TRuntimeNode item) { return item; }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.AggrConcat(pb.AggrConcat(state, pb.NewDataLiteral<NUdf::EDataSlot::String>(" / ")), item); + } + ); + } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 1 / key one / very long value 1 / key one"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 3 / key two / very long value 2 / key two"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 4 / very long key one / very long value 4 / very long key one"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 9 / very long key two / very long value 5 / very long key two"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestDoNotCalculateUnusedInput) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto optionalType = pb.NewOptionalType(dataType); + const auto tupleType = pb.NewTupleType({dataType, optionalType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5"); + + const auto empty = pb.NewDataLiteral<NUdf::EDataSlot::String>(""); + + const auto none = pb.NewEmptyOptional(optionalType); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, none, value1}); + const auto data2 = pb.NewTuple(tupleType, {keyTwo, none, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, none, value3}); + const auto data4 = pb.NewTuple(tupleType, {keyOne, none, value4}); + const auto data5 = pb.NewTuple(tupleType, {keyOne, none, value5}); + const auto data6 = pb.NewTuple(tupleType, {keyOne, none, value1}); + const auto data7 = pb.NewTuple(tupleType, {keyOne, none, value2}); + const auto data8 = pb.NewTuple(tupleType, {keyTwo, none, value3}); + const auto data9 = pb.NewTuple(tupleType, {keyTwo, none, value4}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::String>("ACHTUNG MINEN!"); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Unwrap(pb.Nth(item, 1U), landmine, __FILE__, __LINE__, 0), pb.Nth(item, 2U)}; }), 0ULL, + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) -> TRuntimeNode::TList { + return {items.back(), keys.front(), empty, empty}; + }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { + return {items.back(), state.front(), state[1U], state[2U]}; + }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { + state.insert(state.cbegin(), keys.cbegin(), keys.cend()); + return {pb.NewList(dataType, state)}; + }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { + return pb.Fold1(items.front(), + [&](TRuntimeNode item) { return item; }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.AggrConcat(pb.AggrConcat(state, pb.NewDataLiteral<NUdf::EDataSlot::String>(" / ")), item); + } + ); + } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "key one / value 2 / value 1 / value 5 / value 4"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "key two / value 4 / value 3 / value 3 / value 2"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestDoNotCalculateUnusedOutput) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto optionalType = pb.NewOptionalType(dataType); + const auto tupleType = pb.NewTupleType({dataType, optionalType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5"); + + const auto empty = pb.NewDataLiteral<NUdf::EDataSlot::String>(""); + + const auto none = pb.NewEmptyOptional(optionalType); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, none, value1}); + const auto data2 = pb.NewTuple(tupleType, {keyTwo, none, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, none, value3}); + const auto data4 = pb.NewTuple(tupleType, {keyOne, none, value4}); + const auto data5 = pb.NewTuple(tupleType, {keyOne, none, value5}); + const auto data6 = pb.NewTuple(tupleType, {keyOne, none, value1}); + const auto data7 = pb.NewTuple(tupleType, {keyOne, none, value2}); + const auto data8 = pb.NewTuple(tupleType, {keyTwo, none, value3}); + const auto data9 = pb.NewTuple(tupleType, {keyTwo, none, value4}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::String>("ACHTUNG MINEN!"); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), 0ULL, + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { + return {items[1U], items.back()}; + }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { + return {pb.Concat(state.front(), items[1U]), pb.AggrConcat(pb.AggrConcat(state.back(), pb.NewDataLiteral<NUdf::EDataSlot::String>(", ")), items.back())}; + }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { + return {pb.Unwrap(state.front(), landmine, __FILE__, __LINE__, 0), pb.AggrConcat(pb.AggrConcat(keys.front(), pb.NewDataLiteral<NUdf::EDataSlot::String>(": ")), state.back())}; + }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.back(); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "key one: value 1, value 4, value 5, value 1, value 2"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "key two: value 2, value 3, value 3, value 4"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestThinAllLambdas) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto tupleType = pb.NewTupleType({}); + const auto data = pb.NewTuple({}); + + const auto list = pb.NewList(tupleType, {data, data, data, data}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(list), + [](TRuntimeNode) -> TRuntimeNode::TList { return {}; }), 0ULL, + [](TRuntimeNode::TList items) { return items; }, + [](TRuntimeNode::TList, TRuntimeNode::TList items) { return items; }, + [](TRuntimeNode::TList, TRuntimeNode::TList, TRuntimeNode::TList state) { return state; }, + [](TRuntimeNode::TList, TRuntimeNode::TList state) { return state; }), + [&](TRuntimeNode::TList) { return pb.NewTuple({}); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } +} + +Y_UNIT_TEST_SUITE(TMiniKQLWideCombinerPerfTest) { + Y_UNIT_TEST_LLVM(TestSumDoubleBooleanKeys) { + TSetup<LLVM> setup; + + double positive = 0.0, negative = 0.0; + const auto t = TInstant::Now(); + for (const auto& sample : I8Samples) { + (sample.second > 0.0 ? positive : negative) += sample.second; + } + const auto cppTime = TInstant::Now() - t; + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }), 0ULL, + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {pb.AggrGreater(items.front(), pb.NewDataLiteral(0.0))}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return items; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.front())}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList state) -> TRuntimeNode::TList { return state; }), + [&](TRuntimeNode::TList items) { return items.front(); } + )); + + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items)); + std::transform(I8Samples.cbegin(), I8Samples.cend(), items, [](const std::pair<i8, double> s){ return ToValue<double>(s.second); }); + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto first = value.GetElement(0); + const auto second = value.GetElement(1); + const auto t2 = TInstant::Now(); + + if (first.template Get<double>() > 0.0) { + UNIT_ASSERT_VALUES_EQUAL(first.template Get<double>(), positive); + UNIT_ASSERT_VALUES_EQUAL(second.template Get<double>(), negative); + } else { + UNIT_ASSERT_VALUES_EQUAL(first.template Get<double>(), negative); + UNIT_ASSERT_VALUES_EQUAL(second.template Get<double>(), positive); + } + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleBooleanKeys) { + TSetup<LLVM> setup; + + double pSum = 0.0, nSum = 0.0, pMax = 0.0, nMax = -1000.0, pMin = 1000.0, nMin = 0.0; + const auto t = TInstant::Now(); + for (const auto& sample : I8Samples) { + if (sample.second > 0.0) { + pSum += sample.second; + pMax = std::max(pMax, sample.second); + pMin = std::min(pMin, sample.second); + } else { + nSum += sample.second; + nMax = std::max(nMax, sample.second); + nMin = std::min(nMin, sample.second); + } + } + + const auto cppTime = TInstant::Now() - t; + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }), 0ULL, + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {pb.AggrGreater(items.front(), pb.NewDataLiteral(0.0))}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front(), items.front(), items.front()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { + return {pb.AggrAdd(state.front(), items.front()), pb.AggrMin(state[1U], items.front()), pb.AggrMax(state.back(), items.back()) }; + }, + [&](TRuntimeNode::TList, TRuntimeNode::TList state) -> TRuntimeNode::TList { return state; }), + [&](TRuntimeNode::TList items) { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items)); + std::transform(I8Samples.cbegin(), I8Samples.cend(), items, [](const std::pair<i8, double> s){ return ToValue<double>(s.second); }); + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto first = value.GetElement(0); + const auto second = value.GetElement(1); + const auto t2 = TInstant::Now(); + + if (first.GetElement(0).template Get<double>() > 0.0) { + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(0).template Get<double>(), pSum); + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(1).template Get<double>(), pMin); + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(2).template Get<double>(), pMax); + + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(0).template Get<double>(), nSum); + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(1).template Get<double>(), nMin); + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(2).template Get<double>(), nMax); + } else { + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(0).template Get<double>(), nSum); + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(1).template Get<double>(), nMin); + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(2).template Get<double>(), nMax); + + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(0).template Get<double>(), pSum); + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(1).template Get<double>(), pMin); + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(2).template Get<double>(), pMax); + } + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestSumDoubleSmallKey) { + TSetup<LLVM> setup; + + std::unordered_map<i8, double> expects(201); + const auto t = TInstant::Now(); + for (const auto& sample : I8Samples) { + expects.emplace(sample.first, 0.0).first->second += sample.second; + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<i8, double>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<i8, double> l, const std::pair<i8, double> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i8>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(item, 0U), pb.Nth(item, 1U) }; }), 0ULL, + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.back())}; }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {keys.front(), state.front()}; }), + [&](TRuntimeNode::TList items) { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items)); + for (const auto& sample : I8Samples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[0] = NUdf::TUnboxedValuePod(sample.first); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + two.emplace_back(ptr[i].GetElement(0).template Get<i8>(), ptr[i].GetElement(1).template Get<double>()); + } + + std::sort(two.begin(), two.end(), [](const std::pair<i8, double> l, const std::pair<i8, double> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleSmallKey) { + TSetup<LLVM> setup; + + std::unordered_map<i8, std::array<double, 3U>> expects(201); + const auto t = TInstant::Now(); + for (const auto& sample : I8Samples) { + auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, std::numeric_limits<double>::max(), std::numeric_limits<double>::min()}).first->second; + std::get<0U>(item) += sample.second; + std::get<1U>(item) = std::min(std::get<1U>(item), sample.second); + std::get<2U>(item) = std::max(std::get<2U>(item), sample.second); + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<i8, std::array<double, 3U>>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<i8, std::array<double, 3U>> l, const std::pair<i8, std::array<double, 3U>> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i8>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(item, 0U), pb.Nth(item, 1U) }; }), 0ULL, + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back(), items.back(), items.back()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.back()), pb.AggrMin(state[1U], items.back()), pb.AggrMax(state.back(), items.back())}; }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { state.insert(state.cbegin(), keys.front()); return state; }), + [&](TRuntimeNode::TList items) { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items)); + for (const auto& sample : I8Samples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[0] = NUdf::TUnboxedValuePod(sample.first); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + two.emplace_back(ptr[i].GetElement(0).template Get<i8>(), std::array<double, 3U>{ptr[i].GetElement(1).template Get<double>(), ptr[i].GetElement(2).template Get<double>(), ptr[i].GetElement(3).template Get<double>()}); + } + + std::sort(two.begin(), two.end(), [](const std::pair<i8, std::array<double, 3U>> l, const std::pair<i8, std::array<double, 3U>> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestSumDoubleStringKey) { + TSetup<LLVM> setup; + + std::vector<std::pair<std::string, double>> stringI8Samples(I8Samples.size()); + std::transform(I8Samples.cbegin(), I8Samples.cend(), stringI8Samples.begin(), [](std::pair<i8, double> src){ return std::make_pair(ToString(src.first), src.second); }); + + std::unordered_map<std::string, double> expects(201); + const auto t = TInstant::Now(); + for (const auto& sample : stringI8Samples) { + expects.emplace(sample.first, 0.0).first->second += sample.second; + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<std::string_view, double>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<std::string_view, double> l, const std::pair<std::string_view, double> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<const char*>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(item, 0U), pb.Nth(item, 1U) }; }), 0ULL, + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.back())}; }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {keys.front(), state.front()}; }), + [&](TRuntimeNode::TList items) { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(stringI8Samples.size(), items)); + for (const auto& sample : stringI8Samples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[0] = NUdf::TUnboxedValuePod::Embedded(sample.first); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + two.emplace_back(ptr[i].GetElements()->AsStringRef(), ptr[i].GetElement(1).template Get<double>()); + } + + std::sort(two.begin(), two.end(), [](const std::pair<std::string_view, double> l, const std::pair<std::string_view, double> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleStringKey) { + TSetup<LLVM> setup; + + std::vector<std::pair<std::string, double>> stringI8Samples(I8Samples.size()); + std::transform(I8Samples.cbegin(), I8Samples.cend(), stringI8Samples.begin(), [](std::pair<i8, double> src){ return std::make_pair(ToString(src.first), src.second); }); + + std::unordered_map<std::string, std::array<double, 3U>> expects(201); + const auto t = TInstant::Now(); + for (const auto& sample : stringI8Samples) { + auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, +1E7, -1E7}).first->second; + std::get<0U>(item) += sample.second; + std::get<1U>(item) = std::min(std::get<1U>(item), sample.second); + std::get<2U>(item) = std::max(std::get<2U>(item), sample.second); + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<std::string_view, std::array<double, 3U>>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<std::string_view, std::array<double, 3U>> l, const std::pair<std::string_view, std::array<double, 3U>> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<const char*>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(item, 0U), pb.Nth(item, 1U) }; }), 0ULL, + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back(), items.back(), items.back()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.back()), pb.AggrMin(state[1U], items.back()), pb.AggrMax(state.back(), items.back())}; }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { state.insert(state.cbegin(), keys.front()); return state; }), + [&](TRuntimeNode::TList items) { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(stringI8Samples.size(), items)); + for (const auto& sample : stringI8Samples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[0] = NUdf::TUnboxedValuePod::Embedded(sample.first); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + two.emplace_back(ptr[i].GetElements()->AsStringRef(), std::array<double, 3U>{ptr[i].GetElement(1).template Get<double>(), ptr[i].GetElement(2).template Get<double>(), ptr[i].GetElement(3).template Get<double>()}); + } + + std::sort(two.begin(), two.end(), [](const std::pair<std::string_view, std::array<double, 3U>> l, const std::pair<std::string_view, std::array<double, 3U>> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumTupleKey) { + TSetup<LLVM> setup; + + std::vector<std::pair<std::pair<ui32, std::string>, double>> pairSamples(Ui16Samples.size()); + std::transform(Ui16Samples.cbegin(), Ui16Samples.cend(), pairSamples.begin(), [](std::pair<ui16, double> src){ return std::make_pair(std::make_pair(ui32(src.first / 10U % 100U), ToString(src.first % 10U)), src.second); }); + + struct TPairHash { size_t operator()(const std::pair<ui32, std::string>& p) const { return CombineHashes(std::hash<ui32>()(p.first), std::hash<std::string_view>()(p.second)); } }; + + std::unordered_map<std::pair<ui32, std::string>, std::array<double, 3U>, TPairHash> expects; + const auto t = TInstant::Now(); + for (const auto& sample : pairSamples) { + auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, +1E7, -1E7}).first->second; + std::get<0U>(item) += sample.second; + std::get<1U>(item) = std::min(std::get<1U>(item), sample.second); + std::get<2U>(item) = std::max(std::get<2U>(item), sample.second); + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<std::pair<ui32, std::string>, std::array<double, 3U>>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> l, const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<const char*>::Id)}), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(pb.Nth(item, 0U), 0U), pb.Nth(pb.Nth(item, 0U), 1U), pb.Nth(item, 1U) }; }), 0ULL, + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front(), items[1U]}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back(), items.back(), items.back()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { + return {pb.AggrAdd(state.front(), items.back()), pb.AggrMin(state[1U], items.back()), pb.AggrMax(state.back(), items.back()) }; + }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {keys.front(), keys.back(), state.front(), state[1U], state.back()}; }), + [&](TRuntimeNode::TList items) { return pb.NewTuple({pb.NewTuple({items[0U], items[1U]}), items[2U], items[3U], items[4U]}); } + )); + + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(pairSamples.size(), items)); + for (const auto& sample : pairSamples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + NUdf::TUnboxedValue* keys = nullptr; + pair[0] = graph->GetHolderFactory().CreateDirectArrayHolder(2U, keys); + keys[0] = NUdf::TUnboxedValuePod(sample.first.first); + keys[1] = NUdf::TUnboxedValuePod::Embedded(sample.first.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + const auto elements = ptr[i].GetElements(); + two.emplace_back(std::make_pair(elements[0].GetElement(0).template Get<ui32>(), (elements[0].GetElements()[1]).AsStringRef()), std::array<double, 3U>{elements[1].template Get<double>(), elements[2].template Get<double>(), elements[3].template Get<double>()}); + } + + std::sort(two.begin(), two.end(), [](const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> l, const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestTpch) { + TSetup<LLVM> setup; + + struct TPairHash { size_t operator()(const std::pair<std::string_view, std::string_view>& p) const { return CombineHashes(std::hash<std::string_view>()(p.first), std::hash<std::string_view>()(p.second)); } }; + + std::unordered_map<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>, TPairHash> expects; + const auto t = TInstant::Now(); + for (auto& sample : TpchSamples) { + if (std::get<0U>(sample) <= border) { + const auto& ins = expects.emplace(std::pair<std::string_view, std::string_view>{std::get<1U>(sample), std::get<2U>(sample)}, std::pair<ui64, std::array<double, 5U>>{0ULL, {0., 0., 0., 0., 0.}}); + auto& item = ins.first->second; + ++item.first; + std::get<0U>(item.second) += std::get<3U>(sample); + std::get<1U>(item.second) += std::get<5U>(sample); + std::get<2U>(item.second) += std::get<6U>(sample); + const auto v = std::get<3U>(sample) * (1. - std::get<5U>(sample)); + std::get<3U>(item.second) += v; + std::get<4U>(item.second) += v * (1. + std::get<4U>(sample)); + } + } + for (auto& item : expects) { + std::get<1U>(item.second.second) /= item.second.first; + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<std::pair<std::string, std::string>, std::pair<ui64, std::array<double, 5U>>>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> l, const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui64>::Id), + pb.NewDataType(NUdf::TDataType<const char*>::Id), + pb.NewDataType(NUdf::TDataType<const char*>::Id), + pb.NewDataType(NUdf::TDataType<double>::Id), + pb.NewDataType(NUdf::TDataType<double>::Id), + pb.NewDataType(NUdf::TDataType<double>::Id), + pb.NewDataType(NUdf::TDataType<double>::Id) + })); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner( + pb.WideFilter(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U), pb.Nth(item, 3U), pb.Nth(item, 4U), pb.Nth(item, 5U), pb.Nth(item, 6U)}; }), + [&](TRuntimeNode::TList items) { return pb.AggrLessOrEqual(items.front(), pb.NewDataLiteral<ui64>(border)); } + ), 0ULL, + [&](TRuntimeNode::TList item) -> TRuntimeNode::TList { return {item[1U], item[2U]}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { + const auto price = items[3U]; + const auto disco = items[5U]; + const auto v = pb.Mul(price, pb.Sub(pb.NewDataLiteral<double>(1.), disco)); + return {pb.NewDataLiteral<ui64>(1ULL), price, disco, items[6U], v, pb.Mul(v, pb.Add(pb.NewDataLiteral<double>(1.), items[4U]))}; + }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { + const auto price = items[3U]; + const auto disco = items[5U]; + const auto v = pb.Mul(price, pb.Sub(pb.NewDataLiteral<double>(1.), disco)); + return {pb.Increment(state[0U]), pb.AggrAdd(state[1U], price), pb.AggrAdd(state[2U], disco), pb.AggrAdd(state[3U], items[6U]), pb.AggrAdd(state[4U], v), pb.AggrAdd(state[5U], pb.Mul(v, pb.Add(pb.NewDataLiteral<double>(1.), items[4U])))}; + }, + [&](TRuntimeNode::TList key, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {key.front(), key.back(), state[0U], state[1U], pb.Div(state[2U], state[0U]), state[3U], state[4U], state[5U]}; }), + [&](TRuntimeNode::TList items) { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(TpchSamples.size(), items)); + for (const auto& sample : TpchSamples) { + NUdf::TUnboxedValue* elements = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(7U, elements); + elements[0] = NUdf::TUnboxedValuePod(std::get<0U>(sample)); + elements[1] = NUdf::TUnboxedValuePod::Embedded(std::get<1U>(sample)); + elements[2] = NUdf::TUnboxedValuePod::Embedded(std::get<2U>(sample)); + elements[3] = NUdf::TUnboxedValuePod(std::get<3U>(sample)); + elements[4] = NUdf::TUnboxedValuePod(std::get<4U>(sample)); + elements[5] = NUdf::TUnboxedValuePod(std::get<5U>(sample)); + elements[6] = NUdf::TUnboxedValuePod(std::get<6U>(sample)); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + const auto elements = ptr[i].GetElements(); + two.emplace_back(std::make_pair(elements[0].AsStringRef(), elements[1].AsStringRef()), std::pair<ui64, std::array<double, 5U>>{elements[2].template Get<ui64>(), {elements[3].template Get<double>(), elements[4].template Get<double>(), elements[5].template Get<double>(), elements[6].template Get<double>(), elements[7].template Get<double>()}}); + } + + std::sort(two.begin(), two.end(), [](const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> l, const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } +} +#endif +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 29u +Y_UNIT_TEST_SUITE(TMiniKQLWideLastCombinerTest) { + Y_UNIT_TEST_LLVM(TestLongStringsRefCounting) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto optionalType = pb.NewOptionalType(dataType); + const auto tupleType = pb.NewTupleType({dataType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one"); + const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9"); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, value1}); + + const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3}); + + const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4}); + + const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5}); + const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6}); + const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7}); + const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8}); + const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) -> TRuntimeNode::TList { + return {pb.NewOptional(items.back()), pb.NewOptional(keys.front()), pb.NewEmptyOptional(optionalType), pb.NewEmptyOptional(optionalType)}; + }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { + return {pb.NewOptional(items.back()), state.front(), state[1U], state[2U]}; + }, + [&](TRuntimeNode::TList, TRuntimeNode::TList state) -> TRuntimeNode::TList { + state.erase(state.cbegin()); + return {pb.FlatMap(pb.NewList(optionalType, state), [&](TRuntimeNode item) { return item; } )}; + }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { + return pb.Fold1(items.front(), + [&](TRuntimeNode item) { return item; }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.AggrConcat(pb.AggrConcat(state, pb.NewDataLiteral<NUdf::EDataSlot::String>(" / ")), item); + } + ); + } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "key one"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 2 / key two"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long key one"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 8 / very long value 7 / very long value 6"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestLongStringsPasstroughtRefCounting) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto tupleType = pb.NewTupleType({dataType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one"); + const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9"); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, value1}); + + const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3}); + + const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4}); + + const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5}); + const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6}); + const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7}); + const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8}); + const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) -> TRuntimeNode::TList { + return {items.back(), keys.front(), items.back(), items.front()}; + }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { + return {items.back(), keys.front(), state[2U], state.back()}; + }, + [&](TRuntimeNode::TList, TRuntimeNode::TList state) -> TRuntimeNode::TList { + return state; + }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { + return pb.Fold1(pb.NewList(dataType, items), + [&](TRuntimeNode item) { return item; }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.AggrConcat(pb.AggrConcat(state, pb.NewDataLiteral<NUdf::EDataSlot::String>(" / ")), item); + } + ); + } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 1 / key one / very long value 1 / key one"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 3 / key two / very long value 2 / key two"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 4 / very long key one / very long value 4 / very long key one"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long value 9 / very long key two / very long value 5 / very long key two"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestDoNotCalculateUnusedInput) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto optionalType = pb.NewOptionalType(dataType); + const auto tupleType = pb.NewTupleType({dataType, optionalType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5"); + + const auto empty = pb.NewDataLiteral<NUdf::EDataSlot::String>(""); + + const auto none = pb.NewEmptyOptional(optionalType); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, none, value1}); + const auto data2 = pb.NewTuple(tupleType, {keyTwo, none, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, none, value3}); + const auto data4 = pb.NewTuple(tupleType, {keyOne, none, value4}); + const auto data5 = pb.NewTuple(tupleType, {keyOne, none, value5}); + const auto data6 = pb.NewTuple(tupleType, {keyOne, none, value1}); + const auto data7 = pb.NewTuple(tupleType, {keyOne, none, value2}); + const auto data8 = pb.NewTuple(tupleType, {keyTwo, none, value3}); + const auto data9 = pb.NewTuple(tupleType, {keyTwo, none, value4}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::String>("ACHTUNG MINEN!"); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Unwrap(pb.Nth(item, 1U), landmine, __FILE__, __LINE__, 0), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) -> TRuntimeNode::TList { + return {items.back(), keys.front(), empty, empty}; + }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { + return {items.back(), state.front(), state[1U], state[2U]}; + }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { + state.insert(state.cbegin(), keys.cbegin(), keys.cend()); + return {pb.NewList(dataType, state)}; + }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { + return pb.Fold1(items.front(), + [&](TRuntimeNode item) { return item; }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pb.AggrConcat(pb.AggrConcat(state, pb.NewDataLiteral<NUdf::EDataSlot::String>(" / ")), item); + } + ); + } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "key one / value 2 / value 1 / value 5 / value 4"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "key two / value 4 / value 3 / value 3 / value 2"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestDoNotCalculateUnusedOutput) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto optionalType = pb.NewOptionalType(dataType); + const auto tupleType = pb.NewTupleType({dataType, optionalType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5"); + + const auto empty = pb.NewDataLiteral<NUdf::EDataSlot::String>(""); + + const auto none = pb.NewEmptyOptional(optionalType); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, none, value1}); + const auto data2 = pb.NewTuple(tupleType, {keyTwo, none, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, none, value3}); + const auto data4 = pb.NewTuple(tupleType, {keyOne, none, value4}); + const auto data5 = pb.NewTuple(tupleType, {keyOne, none, value5}); + const auto data6 = pb.NewTuple(tupleType, {keyOne, none, value1}); + const auto data7 = pb.NewTuple(tupleType, {keyOne, none, value2}); + const auto data8 = pb.NewTuple(tupleType, {keyTwo, none, value3}); + const auto data9 = pb.NewTuple(tupleType, {keyTwo, none, value4}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::String>("ACHTUNG MINEN!"); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { + return {items[1U], items.back()}; + }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { + return {pb.Concat(state.front(), items[1U]), pb.AggrConcat(pb.AggrConcat(state.back(), pb.NewDataLiteral<NUdf::EDataSlot::String>(", ")), items.back())}; + }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { + return {pb.Unwrap(state.front(), landmine, __FILE__, __LINE__, 0), pb.AggrConcat(pb.AggrConcat(keys.front(), pb.NewDataLiteral<NUdf::EDataSlot::String>(": ")), state.back())}; + }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.back(); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "key one: value 1, value 4, value 5, value 1, value 2"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "key two: value 2, value 3, value 3, value 4"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestThinAllLambdas) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto tupleType = pb.NewTupleType({}); + const auto data = pb.NewTuple({}); + + const auto list = pb.NewList(tupleType, {data, data, data, data}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(list), + [](TRuntimeNode) -> TRuntimeNode::TList { return {}; }), + [](TRuntimeNode::TList items) { return items; }, + [](TRuntimeNode::TList, TRuntimeNode::TList items) { return items; }, + [](TRuntimeNode::TList, TRuntimeNode::TList, TRuntimeNode::TList state) { return state; }, + [](TRuntimeNode::TList, TRuntimeNode::TList state) { return state; }), + [&](TRuntimeNode::TList) { return pb.NewTuple({}); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } +} + +Y_UNIT_TEST_SUITE(TMiniKQLWideLastCombinerPerfTest) { + Y_UNIT_TEST_LLVM(TestSumDoubleBooleanKeys) { + TSetup<LLVM> setup; + + double positive = 0.0, negative = 0.0; + const auto t = TInstant::Now(); + for (const auto& sample : I8Samples) { + (sample.second > 0.0 ? positive : negative) += sample.second; + } + const auto cppTime = TInstant::Now() - t; + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {pb.AggrGreater(items.front(), pb.NewDataLiteral(0.0))}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return items; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.front())}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList state) -> TRuntimeNode::TList { return state; }), + [&](TRuntimeNode::TList items) { return items.front(); } + )); + + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items)); + std::transform(I8Samples.cbegin(), I8Samples.cend(), items, [](const std::pair<i8, double> s){ return ToValue<double>(s.second); }); + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto first = value.GetElement(0); + const auto second = value.GetElement(1); + const auto t2 = TInstant::Now(); + + if (first.template Get<double>() > 0.0) { + UNIT_ASSERT_VALUES_EQUAL(first.template Get<double>(), positive); + UNIT_ASSERT_VALUES_EQUAL(second.template Get<double>(), negative); + } else { + UNIT_ASSERT_VALUES_EQUAL(first.template Get<double>(), negative); + UNIT_ASSERT_VALUES_EQUAL(second.template Get<double>(), positive); + } + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleBooleanKeys) { + TSetup<LLVM> setup; + + double pSum = 0.0, nSum = 0.0, pMax = 0.0, nMax = -1000.0, pMin = 1000.0, nMin = 0.0; + const auto t = TInstant::Now(); + for (const auto& sample : I8Samples) { + if (sample.second > 0.0) { + pSum += sample.second; + pMax = std::max(pMax, sample.second); + pMin = std::min(pMin, sample.second); + } else { + nSum += sample.second; + nMax = std::max(nMax, sample.second); + nMin = std::min(nMin, sample.second); + } + } + + const auto cppTime = TInstant::Now() - t; + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {pb.AggrGreater(items.front(), pb.NewDataLiteral(0.0))}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front(), items.front(), items.front()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { + return {pb.AggrAdd(state.front(), items.front()), pb.AggrMin(state[1U], items.front()), pb.AggrMax(state.back(), items.back()) }; + }, + [&](TRuntimeNode::TList, TRuntimeNode::TList state) -> TRuntimeNode::TList { return state; }), + [&](TRuntimeNode::TList items) { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items)); + std::transform(I8Samples.cbegin(), I8Samples.cend(), items, [](const std::pair<i8, double> s){ return ToValue<double>(s.second); }); + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto first = value.GetElement(0); + const auto second = value.GetElement(1); + const auto t2 = TInstant::Now(); + + if (first.GetElement(0).template Get<double>() > 0.0) { + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(0).template Get<double>(), pSum); + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(1).template Get<double>(), pMin); + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(2).template Get<double>(), pMax); + + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(0).template Get<double>(), nSum); + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(1).template Get<double>(), nMin); + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(2).template Get<double>(), nMax); + } else { + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(0).template Get<double>(), nSum); + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(1).template Get<double>(), nMin); + UNIT_ASSERT_VALUES_EQUAL(first.GetElement(2).template Get<double>(), nMax); + + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(0).template Get<double>(), pSum); + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(1).template Get<double>(), pMin); + UNIT_ASSERT_VALUES_EQUAL(second.GetElement(2).template Get<double>(), pMax); + } + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestSumDoubleSmallKey) { + TSetup<LLVM> setup; + + std::unordered_map<i8, double> expects(201); + const auto t = TInstant::Now(); + for (const auto& sample : I8Samples) { + expects.emplace(sample.first, 0.0).first->second += sample.second; + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<i8, double>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<i8, double> l, const std::pair<i8, double> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i8>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(item, 0U), pb.Nth(item, 1U) }; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.back())}; }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {keys.front(), state.front()}; }), + [&](TRuntimeNode::TList items) { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items)); + for (const auto& sample : I8Samples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[0] = NUdf::TUnboxedValuePod(sample.first); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + two.emplace_back(ptr[i].GetElement(0).template Get<i8>(), ptr[i].GetElement(1).template Get<double>()); + } + + std::sort(two.begin(), two.end(), [](const std::pair<i8, double> l, const std::pair<i8, double> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleSmallKey) { + TSetup<LLVM> setup; + + std::unordered_map<i8, std::array<double, 3U>> expects(201); + const auto t = TInstant::Now(); + for (const auto& sample : I8Samples) { + auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, std::numeric_limits<double>::max(), std::numeric_limits<double>::min()}).first->second; + std::get<0U>(item) += sample.second; + std::get<1U>(item) = std::min(std::get<1U>(item), sample.second); + std::get<2U>(item) = std::max(std::get<2U>(item), sample.second); + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<i8, std::array<double, 3U>>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<i8, std::array<double, 3U>> l, const std::pair<i8, std::array<double, 3U>> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i8>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(item, 0U), pb.Nth(item, 1U) }; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back(), items.back(), items.back()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.back()), pb.AggrMin(state[1U], items.back()), pb.AggrMax(state.back(), items.back())}; }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { state.insert(state.cbegin(), keys.front()); return state; }), + [&](TRuntimeNode::TList items) { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items)); + for (const auto& sample : I8Samples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[0] = NUdf::TUnboxedValuePod(sample.first); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + two.emplace_back(ptr[i].GetElement(0).template Get<i8>(), std::array<double, 3U>{ptr[i].GetElement(1).template Get<double>(), ptr[i].GetElement(2).template Get<double>(), ptr[i].GetElement(3).template Get<double>()}); + } + + std::sort(two.begin(), two.end(), [](const std::pair<i8, std::array<double, 3U>> l, const std::pair<i8, std::array<double, 3U>> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestSumDoubleStringKey) { + TSetup<LLVM> setup; + + std::vector<std::pair<std::string, double>> stringI8Samples(I8Samples.size()); + std::transform(I8Samples.cbegin(), I8Samples.cend(), stringI8Samples.begin(), [](std::pair<i8, double> src){ return std::make_pair(ToString(src.first), src.second); }); + + std::unordered_map<std::string, double> expects(201); + const auto t = TInstant::Now(); + for (const auto& sample : stringI8Samples) { + expects.emplace(sample.first, 0.0).first->second += sample.second; + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<std::string_view, double>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<std::string_view, double> l, const std::pair<std::string_view, double> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<const char*>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(item, 0U), pb.Nth(item, 1U) }; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.back())}; }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {keys.front(), state.front()}; }), + [&](TRuntimeNode::TList items) { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(stringI8Samples.size(), items)); + for (const auto& sample : stringI8Samples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[0] = NUdf::TUnboxedValuePod::Embedded(sample.first); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + two.emplace_back(ptr[i].GetElements()->AsStringRef(), ptr[i].GetElement(1).template Get<double>()); + } + + std::sort(two.begin(), two.end(), [](const std::pair<std::string_view, double> l, const std::pair<std::string_view, double> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleStringKey) { + TSetup<LLVM> setup; + + std::vector<std::pair<std::string, double>> stringI8Samples(I8Samples.size()); + std::transform(I8Samples.cbegin(), I8Samples.cend(), stringI8Samples.begin(), [](std::pair<i8, double> src){ return std::make_pair(ToString(src.first), src.second); }); + + std::unordered_map<std::string, std::array<double, 3U>> expects(201); + const auto t = TInstant::Now(); + for (const auto& sample : stringI8Samples) { + auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, +1E7, -1E7}).first->second; + std::get<0U>(item) += sample.second; + std::get<1U>(item) = std::min(std::get<1U>(item), sample.second); + std::get<2U>(item) = std::max(std::get<2U>(item), sample.second); + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<std::string_view, std::array<double, 3U>>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<std::string_view, std::array<double, 3U>> l, const std::pair<std::string_view, std::array<double, 3U>> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<const char*>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(item, 0U), pb.Nth(item, 1U) }; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back(), items.back(), items.back()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.back()), pb.AggrMin(state[1U], items.back()), pb.AggrMax(state.back(), items.back())}; }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { state.insert(state.cbegin(), keys.front()); return state; }), + [&](TRuntimeNode::TList items) { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(stringI8Samples.size(), items)); + for (const auto& sample : stringI8Samples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[0] = NUdf::TUnboxedValuePod::Embedded(sample.first); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + two.emplace_back(ptr[i].GetElements()->AsStringRef(), std::array<double, 3U>{ptr[i].GetElement(1).template Get<double>(), ptr[i].GetElement(2).template Get<double>(), ptr[i].GetElement(3).template Get<double>()}); + } + + std::sort(two.begin(), two.end(), [](const std::pair<std::string_view, std::array<double, 3U>> l, const std::pair<std::string_view, std::array<double, 3U>> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestMinMaxSumTupleKey) { + TSetup<LLVM> setup; + + std::vector<std::pair<std::pair<ui32, std::string>, double>> pairSamples(Ui16Samples.size()); + std::transform(Ui16Samples.cbegin(), Ui16Samples.cend(), pairSamples.begin(), [](std::pair<ui16, double> src){ return std::make_pair(std::make_pair(ui32(src.first / 10U % 100U), ToString(src.first % 10U)), src.second); }); + + struct TPairHash { size_t operator()(const std::pair<ui32, std::string>& p) const { return CombineHashes(std::hash<ui32>()(p.first), std::hash<std::string_view>()(p.second)); } }; + + std::unordered_map<std::pair<ui32, std::string>, std::array<double, 3U>, TPairHash> expects; + const auto t = TInstant::Now(); + for (const auto& sample : pairSamples) { + auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, +1E7, -1E7}).first->second; + std::get<0U>(item) += sample.second; + std::get<1U>(item) = std::min(std::get<1U>(item), sample.second); + std::get<2U>(item) = std::max(std::get<2U>(item), sample.second); + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<std::pair<ui32, std::string>, std::array<double, 3U>>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> l, const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<const char*>::Id)}), pb.NewDataType(NUdf::TDataType<double>::Id)})); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(pb.Nth(item, 0U), 0U), pb.Nth(pb.Nth(item, 0U), 1U), pb.Nth(item, 1U) }; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front(), items[1U]}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back(), items.back(), items.back()}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { + return {pb.AggrAdd(state.front(), items.back()), pb.AggrMin(state[1U], items.back()), pb.AggrMax(state.back(), items.back()) }; + }, + [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {keys.front(), keys.back(), state.front(), state[1U], state.back()}; }), + [&](TRuntimeNode::TList items) { return pb.NewTuple({pb.NewTuple({items[0U], items[1U]}), items[2U], items[3U], items[4U]}); } + )); + + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(pairSamples.size(), items)); + for (const auto& sample : pairSamples) { + NUdf::TUnboxedValue* pair = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair); + pair[1] = NUdf::TUnboxedValuePod(sample.second); + NUdf::TUnboxedValue* keys = nullptr; + pair[0] = graph->GetHolderFactory().CreateDirectArrayHolder(2U, keys); + keys[0] = NUdf::TUnboxedValuePod(sample.first.first); + keys[1] = NUdf::TUnboxedValuePod::Embedded(sample.first.second); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + const auto elements = ptr[i].GetElements(); + two.emplace_back(std::make_pair(elements[0].GetElement(0).template Get<ui32>(), (elements[0].GetElements()[1]).AsStringRef()), std::array<double, 3U>{elements[1].template Get<double>(), elements[2].template Get<double>(), elements[3].template Get<double>()}); + } + + std::sort(two.begin(), two.end(), [](const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> l, const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } + + Y_UNIT_TEST_LLVM(TestTpch) { + TSetup<LLVM> setup; + + struct TPairHash { size_t operator()(const std::pair<std::string_view, std::string_view>& p) const { return CombineHashes(std::hash<std::string_view>()(p.first), std::hash<std::string_view>()(p.second)); } }; + + std::unordered_map<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>, TPairHash> expects; + const auto t = TInstant::Now(); + for (auto& sample : TpchSamples) { + if (std::get<0U>(sample) <= border) { + const auto& ins = expects.emplace(std::pair<std::string_view, std::string_view>{std::get<1U>(sample), std::get<2U>(sample)}, std::pair<ui64, std::array<double, 5U>>{0ULL, {0., 0., 0., 0., 0.}}); + auto& item = ins.first->second; + ++item.first; + std::get<0U>(item.second) += std::get<3U>(sample); + std::get<1U>(item.second) += std::get<5U>(sample); + std::get<2U>(item.second) += std::get<6U>(sample); + const auto v = std::get<3U>(sample) * (1. - std::get<5U>(sample)); + std::get<3U>(item.second) += v; + std::get<4U>(item.second) += v * (1. + std::get<4U>(sample)); + } + } + for (auto& item : expects) { + std::get<1U>(item.second.second) /= item.second.first; + } + const auto cppTime = TInstant::Now() - t; + + std::vector<std::pair<std::pair<std::string, std::string>, std::pair<ui64, std::array<double, 5U>>>> one, two; + one.reserve(expects.size()); + two.reserve(expects.size()); + + one.insert(one.cend(), expects.cbegin(), expects.cend()); + std::sort(one.begin(), one.end(), [](const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> l, const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> r){ return l.first < r.first; }); + + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto listType = pb.NewListType(pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui64>::Id), + pb.NewDataType(NUdf::TDataType<const char*>::Id), + pb.NewDataType(NUdf::TDataType<const char*>::Id), + pb.NewDataType(NUdf::TDataType<double>::Id), + pb.NewDataType(NUdf::TDataType<double>::Id), + pb.NewDataType(NUdf::TDataType<double>::Id), + pb.NewDataType(NUdf::TDataType<double>::Id) + })); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner( + pb.WideFilter(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U), pb.Nth(item, 3U), pb.Nth(item, 4U), pb.Nth(item, 5U), pb.Nth(item, 6U)}; }), + [&](TRuntimeNode::TList items) { return pb.AggrLessOrEqual(items.front(), pb.NewDataLiteral<ui64>(border)); } + ), + [&](TRuntimeNode::TList item) -> TRuntimeNode::TList { return {item[1U], item[2U]}; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { + const auto price = items[3U]; + const auto disco = items[5U]; + const auto v = pb.Mul(price, pb.Sub(pb.NewDataLiteral<double>(1.), disco)); + return {pb.NewDataLiteral<ui64>(1ULL), price, disco, items[6U], v, pb.Mul(v, pb.Add(pb.NewDataLiteral<double>(1.), items[4U]))}; + }, + [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { + const auto price = items[3U]; + const auto disco = items[5U]; + const auto v = pb.Mul(price, pb.Sub(pb.NewDataLiteral<double>(1.), disco)); + return {pb.Increment(state[0U]), pb.AggrAdd(state[1U], price), pb.AggrAdd(state[2U], disco), pb.AggrAdd(state[3U], items[6U]), pb.AggrAdd(state[4U], v), pb.AggrAdd(state[5U], pb.Mul(v, pb.Add(pb.NewDataLiteral<double>(1.), items[4U])))}; + }, + [&](TRuntimeNode::TList key, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {key.front(), key.back(), state[0U], state[1U], pb.Div(state[2U], state[0U]), state[3U], state[4U], state[5U]}; }), + [&](TRuntimeNode::TList items) { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn, {list}); + NUdf::TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(TpchSamples.size(), items)); + for (const auto& sample : TpchSamples) { + NUdf::TUnboxedValue* elements = nullptr; + *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(7U, elements); + elements[0] = NUdf::TUnboxedValuePod(std::get<0U>(sample)); + elements[1] = NUdf::TUnboxedValuePod::Embedded(std::get<1U>(sample)); + elements[2] = NUdf::TUnboxedValuePod::Embedded(std::get<2U>(sample)); + elements[3] = NUdf::TUnboxedValuePod(std::get<3U>(sample)); + elements[4] = NUdf::TUnboxedValuePod(std::get<4U>(sample)); + elements[5] = NUdf::TUnboxedValuePod(std::get<5U>(sample)); + elements[6] = NUdf::TUnboxedValuePod(std::get<6U>(sample)); + } + + const auto t1 = TInstant::Now(); + const auto& value = graph->GetValue(); + const auto t2 = TInstant::Now(); + + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size()); + + const auto ptr = value.GetElements(); + for (size_t i = 0ULL; i < expects.size(); ++i) { + const auto elements = ptr[i].GetElements(); + two.emplace_back(std::make_pair(elements[0].AsStringRef(), elements[1].AsStringRef()), std::pair<ui64, std::array<double, 5U>>{elements[2].template Get<ui64>(), {elements[3].template Get<double>(), elements[4].template Get<double>(), elements[5].template Get<double>(), elements[6].template Get<double>(), elements[7].template Get<double>()}}); + } + + std::sort(two.begin(), two.end(), [](const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> l, const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> r){ return l.first < r.first; }); + UNIT_ASSERT_VALUES_EQUAL(one, two); + + Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl; + } +} +#endif +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_condense_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_condense_ut.cpp new file mode 100644 index 00000000000..f7fdc9e0d4b --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_condense_ut.cpp @@ -0,0 +1,173 @@ +#include "mkql_computation_node_ut.h" +#include <ydb/library/yql/minikql/mkql_runtime_version.h> + +namespace NKikimr { +namespace NMiniKQL { +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u +Y_UNIT_TEST_SUITE(TMiniKQLWideCondense1Test) { + Y_UNIT_TEST_LLVM(TestConcatItemsToKey) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto tupleType = pb.NewTupleType({dataType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one"); + const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9"); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, value1}); + + const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3}); + + const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4}); + + const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5}); + const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6}); + const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7}); + const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8}); + const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCondense1(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { + return {items.front(), pb.AggrConcat(pb.AggrConcat(items.front(), pb.NewDataLiteral<NUdf::EDataSlot::String>(": ")), items.back())}; + }, + [&](TRuntimeNode::TList items, TRuntimeNode::TList state) { + return pb.AggrNotEquals(items.front(), state.front()); + }, + [&](TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { + return {state.front(), pb.AggrConcat(pb.AggrConcat(state.back(), pb.NewDataLiteral<NUdf::EDataSlot::String>(", ")), items.back())}; + }), + [&](TRuntimeNode::TList items) { return items.back(); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "key one: very long value 1"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "key two: very long value 2, very long value 3"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long key one: very long value 4"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "very long key two: very long value 5, very long value 6, very long value 7, very long value 8, very long value 9"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestSwitchByBoolFieldAndDontUseKey) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto boolType = pb.NewDataType(NUdf::TDataType<bool>::Id); + const auto tupleType = pb.NewTupleType({pb.NewOptionalType(dataType), dataType, boolType}); + + const auto key0 = pb.NewEmptyOptional(pb.NewOptionalType(dataType)); + const auto key1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("one")); + const auto key2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("two")); + + const auto trueVal = pb.NewDataLiteral<bool>(true); + const auto falseVal = pb.NewDataLiteral<bool>(false); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 9"); + + const auto data1 = pb.NewTuple(tupleType, {key0, value1, trueVal}); + const auto data2 = pb.NewTuple(tupleType, {key1, value2, falseVal}); + const auto data3 = pb.NewTuple(tupleType, {key2, value3, falseVal}); + const auto data4 = pb.NewTuple(tupleType, {key0, value4, trueVal}); + const auto data5 = pb.NewTuple(tupleType, {key1, value5, falseVal}); + const auto data6 = pb.NewTuple(tupleType, {key2, value6, falseVal}); + const auto data7 = pb.NewTuple(tupleType, {key0, value7, falseVal}); + const auto data8 = pb.NewTuple(tupleType, {key1, value8, falseVal}); + const auto data9 = pb.NewTuple(tupleType, {key2, value9, trueVal}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::String>("ACHTUNG MINEN!"); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCondense1(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Unwrap(pb.Nth(item, 0U), landmine, __FILE__, __LINE__, 0), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { + return {items[1U]}; + }, + [&](TRuntimeNode::TList items, TRuntimeNode::TList) { + return items.back(); + }, + [&](TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { + return {pb.AggrConcat(pb.AggrConcat(state.front(), pb.NewDataLiteral<NUdf::EDataSlot::String>("; ")), items[1U])}; + }), + [&](TRuntimeNode::TList items) { return items.front(); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 1; value 2; value 3"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 4; value 5; value 6; value 7; value 8"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "value 9"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestThinAllLambdas) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto tupleType = pb.NewTupleType({}); + + const auto data = pb.NewTuple({}); + + const auto list = pb.NewList(tupleType, {data, data, data, data}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCondense1(pb.ExpandMap(pb.ToFlow(list), + [](TRuntimeNode) -> TRuntimeNode::TList { return {}; }), + [](TRuntimeNode::TList items) { return items; }, + [&](TRuntimeNode::TList, TRuntimeNode::TList) { return pb.NewDataLiteral<bool>(true); }, + [](TRuntimeNode::TList, TRuntimeNode::TList state) { return state;}), + [&](TRuntimeNode::TList) { return pb.NewTuple({}); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } +} +#endif +} +} + diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_filter_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_filter_ut.cpp new file mode 100644 index 00000000000..5d5a9e7aa73 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_filter_ut.cpp @@ -0,0 +1,386 @@ +#include "mkql_computation_node_ut.h" +#include <ydb/library/yql/minikql/mkql_runtime_version.h> + +namespace NKikimr { +namespace NMiniKQL { +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u +Y_UNIT_TEST_SUITE(TMiniKQLWideFilterTest) { + Y_UNIT_TEST_LLVM(TestPredicateExpression) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3}); + + const auto pgmReturn = pb.FromFlow( + pb.NarrowMap( + pb.WideFilter( + pb.ExpandMap( + pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; } + ), + [&](TRuntimeNode::TList items) -> TRuntimeNode { + const auto v = pb.If( + pb.Exists(items.front()), + pb.NewOptional(pb.NewDataLiteral<bool>(true)), + pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id) + ); + return pb.Coalesce(v, pb.NewDataLiteral<bool>(false)); + } + ), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); } + ) + ); + + const auto graph = setup.BuildGraph(pgmReturn); + NUdf::TUnboxedValue value = graph->GetValue(); + + NUdf::TUnboxedValue v; + UNIT_ASSERT_VALUES_EQUAL(value.Fetch(v), NUdf::EFetchStatus::Ok); + UNIT_ASSERT_VALUES_EQUAL(v.GetElement(0).template Get<i32>(), 2); + UNIT_ASSERT_VALUES_EQUAL(v.GetElement(1).template Get<i32>(), -2); + + UNIT_ASSERT_VALUES_EQUAL(value.Fetch(v), NUdf::EFetchStatus::Ok); + UNIT_ASSERT_VALUES_EQUAL(v.GetElement(0).template Get<i32>(), 3); + UNIT_ASSERT_VALUES_EQUAL(v.GetElement(1).template Get<i32>(), -3); + + UNIT_ASSERT_VALUES_EQUAL(value.Fetch(v), NUdf::EFetchStatus::Finish); + } + + Y_UNIT_TEST_LLVM(TestCheckedFieldPasstrought) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideFilter(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.Exists(items[1U]); }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 4); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4); + UNIT_ASSERT(!item.GetElement(2)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestCheckedFieldUnusedAfter) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideFilter(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.Exists(items[1U]); }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple({items.back(), items.front()}); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -2); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestDotCalculateUnusedField) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::String>("ACHTUNG MINEN!"); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideFilter(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Unwrap(pb.Nth(item, 2U), landmine, __FILE__, __LINE__, 0)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.Exists(items[1U]); }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.front(); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 4); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestWithLimitCheckedFieldUsedAfter) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(9)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(7)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(6)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideFilter(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + pb.NewDataLiteral<ui64>(2ULL), [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.Exists(items.front()); }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.AggrAdd(items.back(), items.front()); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 8); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 4); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestWithLimitCheckedFieldUnusedAfter) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideFilter(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + pb.NewDataLiteral<ui64>(2ULL), [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.Exists(items.front()); }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.back(); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -3); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestTakeWhile) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTakeWhile(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.Exists(items.front()); }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -1); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestTakeWhileInclusive) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTakeWhileInclusive(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.Exists(items.front()); }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -2); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestSkipWhile) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideSkipWhile(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.AggrGreater(items.back(), pb.NewOptional(pb.NewDataLiteral<i32>(-3))); }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 3); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 4); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4); + UNIT_ASSERT(!item.GetElement(2)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestSkipWhileInclusive) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideSkipWhileInclusive(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.AggrGreater(items.back(), pb.NewOptional(pb.NewDataLiteral<i32>(-3))); }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 4); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4); + UNIT_ASSERT(!item.GetElement(2)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFilterByBooleanField) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, pb.NewDataType(NUdf::TDataType<bool>::Id), dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewDataLiteral(true), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewDataLiteral(false), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewDataLiteral(true), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewDataLiteral(false), pb.NewEmptyOptional(dataType)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideFilter(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return items[1U]; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple({items.back(), items.front()}); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -3); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +3); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } +} +#endif +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_map_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_map_ut.cpp new file mode 100644 index 00000000000..4add5f00ea2 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_map_ut.cpp @@ -0,0 +1,253 @@ +#include "mkql_computation_node_ut.h" +#include <ydb/library/yql/minikql/mkql_runtime_version.h> + +namespace NKikimr { +namespace NMiniKQL { +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u +Y_UNIT_TEST_SUITE(TMiniKQLWideMapTest) { + Y_UNIT_TEST_LLVM(TestSimpleSwap) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple({items[2U], items[1U], items[0U] }); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -1); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -2); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2); + UNIT_ASSERT(!item.GetElement(2)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -3); + UNIT_ASSERT(!item.GetElement(1)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 3); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 4); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestThinLambda) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType)}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4))}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideMap(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode) -> TRuntimeNode::TList { return {}; }), + [&](TRuntimeNode::TList items) { return items; }), + [&](TRuntimeNode::TList) -> TRuntimeNode { return pb.NewTuple({}); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestWideMap) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideMap(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {pb.AggrMin(items[0], items[1]), pb.AggrMax(items[1], items[2]), pb.AggrAdd(items[0], items[2])}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 2); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -2); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 3); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -3); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 0); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 4); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 4); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestDotCalculateUnusedField) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewOptional(pb.NewDataLiteral<i32>(0)), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-4))}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::String>("ACHTUNG MINEN!"); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideMap(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {pb.Mul(items.front(), items.back()), pb.Unwrap(items[1], landmine, __FILE__, __LINE__, 0), pb.Add(items.front(), items.back())}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple({items.back(), items.front()}); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -1); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -9); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 0); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -16); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestPasstroughtFieldSplitAsIs) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewOptional(pb.NewDataLiteral<i32>(-5)), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-4)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewOptional(pb.NewDataLiteral<i32>(-7)), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-4))}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideMap(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items[1U], pb.Mul(items.front(), items.back()), items[1U]}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -5); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -1); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -5); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -4); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -4); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -7); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -9); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -7); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -16); + UNIT_ASSERT(!item.GetElement(2)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestFieldBothWayPasstroughtAndArg) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)); + const auto tupleType = pb.NewTupleType({dataType, dataType, dataType}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewOptional(pb.NewDataLiteral<i32>(-5)), pb.NewOptional(pb.NewDataLiteral<i32>(-1))}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-4)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewOptional(pb.NewDataLiteral<i32>(-7)), pb.NewOptional(pb.NewDataLiteral<i32>(-3))}); + const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-4))}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideMap(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items[1U], pb.Sub(items.front(), items.back()), pb.Minus(items[1U])}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -5); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 5); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -4); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 4); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -7); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 6); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 7); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT(!item.GetElement(0)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 8); + UNIT_ASSERT(!item.GetElement(2)); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } +} +#endif +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_nodes_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_nodes_ut.cpp new file mode 100644 index 00000000000..87158beed65 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_nodes_ut.cpp @@ -0,0 +1,124 @@ +#include "mkql_computation_node_ut.h" +#include <ydb/library/yql/minikql/mkql_runtime_version.h> + +namespace NKikimr { +namespace NMiniKQL { +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u +Y_UNIT_TEST_SUITE(TMiniKQLWideNodesTest) { + // TDOD: fixme +#if 0 + Y_UNIT_TEST_LLVM(TestWideDiscard) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000"); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200"); + const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300"); + const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3}); + + const auto pgmReturn = pb.FromFlow(pb.Discard(pb.ExpandMap(pb.ToFlow(list), [](TRuntimeNode) { return TRuntimeNode::TList(); }))); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } +#endif + + Y_UNIT_TEST_LLVM(TestDiscard) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000"); + const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100"); + const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200"); + const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300"); + const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto list = pb.NewList(dataType, {data0, data1, data2, data3}); + + const auto pgmReturn = pb.FromFlow(pb.Discard(pb.ToFlow(list))); + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item)); + } + + Y_UNIT_TEST_LLVM(TestTakeOverSource) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.Take(pb.Source(), pb.NewDataLiteral<ui64>(666ULL)), [&](TRuntimeNode::TList) { return pb.NewTuple({}); } )); + + const auto graph = setup.BuildGraph(pgmReturn); + UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 666ULL); + } + + Y_UNIT_TEST_LLVM(TestSkipAndTake) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list = pb.ListFromRange(pb.NewDataLiteral<ui32>(100U), pb.NewDataLiteral<ui32>(666U), pb.NewDataLiteral<ui32>(3U)); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.Take(pb.Skip(pb.ExpandMap(pb.ToFlow(pb.Enumerate(list)), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 1U), pb.Nth(item, 0U)}; }), + pb.NewDataLiteral<ui64>(42ULL)), pb.NewDataLiteral<ui64>(4ULL)), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple({items.back(), items.front()}); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 42); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 226); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 43); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 229); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 44); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 232); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 45); + UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 235); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TestDoNotCalculateSkipped) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto list = pb.ListFromRange(pb.NewDataLiteral<ui64>(100ULL), pb.NewDataLiteral<ui64>(135ULL), pb.NewDataLiteral<ui64>(5ULL)); + + const auto trap = pb.NewDataLiteral<NUdf::EDataSlot::String>("IT'S A TRAP!"); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.Skip(pb.WideMap(pb.ExpandMap(pb.ToFlow(pb.Enumerate(list)), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 1U), pb.Nth(item, 0U)}; }), + [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {pb.Unwrap(pb.Div(items.front(), items.back()), trap, __FILE__, __LINE__, 0)}; }), + pb.NewDataLiteral<ui64>(3ULL)), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.front(); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 38ULL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 30ULL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 25ULL); + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 21ULL); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + +} +#endif +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_stream_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_stream_ut.cpp new file mode 100644 index 00000000000..093ae70d352 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_stream_ut.cpp @@ -0,0 +1,58 @@ +#include "mkql_computation_node_ut.h" + +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> + +namespace NKikimr { +namespace NMiniKQL { + +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 36u +Y_UNIT_TEST_SUITE(TMiniKQLWideStreamTest) { + +Y_UNIT_TEST_LLVM(TestSimple) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id); + const auto tupleType = pb.NewTupleType({ui64Type, ui64Type}); + + const auto data1 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(1), pb.NewDataLiteral<ui64>(10)}); + const auto data2 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(2), pb.NewDataLiteral<ui64>(20)}); + const auto data3 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(3), pb.NewDataLiteral<ui64>(30)}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3}); + const auto flow = pb.ToFlow(list); + + const auto wideFlow = pb.ExpandMap(flow, [&](TRuntimeNode item) -> TRuntimeNode::TList { + return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; + }); + + const auto wideStream = pb.FromFlow(wideFlow); + const auto newWideFlow = pb.ToFlow(wideStream); + + const auto narrowFlow = pb.NarrowMap(newWideFlow, [&](TRuntimeNode::TList items) -> TRuntimeNode { + return pb.Sub(items[1], items[0]); + }); + const auto pgmReturn = pb.Collect(narrowFlow); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 9); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 18); + + UNIT_ASSERT(iterator.Next(item)); + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 27); + + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); +} +} + +#endif + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_top_sort_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_top_sort_ut.cpp new file mode 100644 index 00000000000..b52b6175878 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_top_sort_ut.cpp @@ -0,0 +1,651 @@ +#include "mkql_computation_node_ut.h" +#include <ydb/library/yql/minikql/mkql_runtime_version.h> + +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> + +#include <cstring> + +namespace NKikimr { +namespace NMiniKQL { + +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 33u +Y_UNIT_TEST_SUITE(TMiniKQLWideTopTest) { + Y_UNIT_TEST_LLVM(TopByFirstKeyAsc) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto tupleType = pb.NewTupleType({dataType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one"); + const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9"); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, value1}); + + const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3}); + + const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4}); + + const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5}); + const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6}); + const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7}); + const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8}); + const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTop(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.NewDataLiteral<ui64>(4ULL), {{0U, pb.NewDataLiteral<bool>(true)}}), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(tupleType, items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key one"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 4"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 3"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 2"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key one"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 1"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TopByFirstKeyDesc) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto tupleType = pb.NewTupleType({dataType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one"); + const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9"); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, value1}); + + const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3}); + + const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4}); + + const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5}); + const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6}); + const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7}); + const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8}); + const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTop(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.NewDataLiteral<ui64>(6ULL), {{0U, pb.NewDataLiteral<bool>(false)}}), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(tupleType, items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key one"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 4"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 7"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 5"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 6"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 8"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 9"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TopBySecondKeyAsc) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto tupleType = pb.NewTupleType({dataType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one"); + const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9"); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, value1}); + + const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3}); + + const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4}); + + const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5}); + const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6}); + const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7}); + const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8}); + const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTop(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.NewDataLiteral<ui64>(3ULL), {{1U, pb.NewDataLiteral<bool>(true)}}), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(tupleType, items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 3"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 2"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key one"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 1"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TopBySecondKeyDesc) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto tupleType = pb.NewTupleType({dataType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one"); + const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9"); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, value1}); + + const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3}); + + const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4}); + + const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5}); + const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6}); + const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7}); + const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8}); + const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTop(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.NewDataLiteral<ui64>(2ULL), {{1U, pb.NewDataLiteral<bool>(false)}}), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(tupleType, items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 8"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 9"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TopSortByFirstSecondAscDesc) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto tupleType = pb.NewTupleType({dataType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one"); + const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9"); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, value1}); + + const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3}); + + const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4}); + + const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5}); + const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6}); + const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7}); + const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8}); + const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTopSort(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.NewDataLiteral<ui64>(4ULL), {{0U, pb.NewDataLiteral<bool>(true)}, {1U, pb.NewDataLiteral<bool>(false)}}), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(tupleType, items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key one"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 1"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 3"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 2"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key one"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 4"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TopSortByFirstSecondDescAsc) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto tupleType = pb.NewTupleType({dataType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one"); + const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9"); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, value1}); + + const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3}); + + const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4}); + + const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5}); + const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6}); + const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7}); + const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8}); + const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTopSort(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.NewDataLiteral<ui64>(6ULL), {{0U, pb.NewDataLiteral<bool>(false)}, {1U, pb.NewDataLiteral<bool>(true)}}), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(tupleType, items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 5"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 6"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 7"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 8"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 9"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key one"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 4"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TopSortBySecondFirstAscDesc) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto tupleType = pb.NewTupleType({dataType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one"); + const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9"); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, value1}); + + const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3}); + + const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4}); + + const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5}); + const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6}); + const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7}); + const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8}); + const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTopSort(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.NewDataLiteral<ui64>(4ULL), {{1U, pb.NewDataLiteral<bool>(true)}, {0U, pb.NewDataLiteral<bool>(false)}}), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(tupleType, items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key one"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 1"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 2"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 3"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key one"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 4"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TopSortBySecondFirstDescAsc) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto tupleType = pb.NewTupleType({dataType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one"); + const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9"); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, value1}); + + const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3}); + + const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4}); + + const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5}); + const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6}); + const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7}); + const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8}); + const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTopSort(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + pb.NewDataLiteral<ui64>(6ULL), {{1U, pb.NewDataLiteral<bool>(false)}, {0U, pb.NewDataLiteral<bool>(true)}}), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(tupleType, items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 9"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 8"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 7"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 6"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 5"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key one"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 4"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } + + Y_UNIT_TEST_LLVM(TopSortLargeList) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto minusday = i64(-24LL * 60LL * 60LL * 1000000LL); // -1 Day + const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&minusday, sizeof(minusday))); + const auto list = pb.ListFromRange(pb.NewTzDataLiteral<NUdf::TTzDate>(30000u, 42u), pb.NewTzDataLiteral<NUdf::TTzDate>(10000u, 42u), step); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTopSort(pb.ExpandMap(pb.ToFlow(pb.Enumerate(list)), + [&](TRuntimeNode item) -> TRuntimeNode::TList { + const auto utf = pb.ToString<true>(pb.Nth(item, 1U)); + const auto day = pb.StrictFromString(pb.Substring(utf, pb.NewDataLiteral<ui32>(8U), pb.NewDataLiteral<ui32>(2U)), pb.NewDataType(NUdf::EDataSlot::Uint8)); + return {pb.Nth(item, 0U), utf, day, pb.Nth(item, 1U)}; + }), + pb.NewDataLiteral<ui64>(7ULL), {{2U, pb.NewDataLiteral<bool>(true)}, {0U, pb.NewDataLiteral<bool>(false)}}), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return items[1]; } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "1997-06-01,Africa/Mbabane"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "1997-07-01,Africa/Mbabane"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "1997-08-01,Africa/Mbabane"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "1997-09-01,Africa/Mbabane"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "1997-10-01,Africa/Mbabane"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "1997-11-01,Africa/Mbabane"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item, "1997-12-01,Africa/Mbabane"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } +} +#endif + +#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 34u +Y_UNIT_TEST_SUITE(TMiniKQLWideSortTest) { + Y_UNIT_TEST_LLVM(SortByFirstKeyAsc) { + TSetup<LLVM> setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id); + const auto tupleType = pb.NewTupleType({dataType, dataType}); + + const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one"); + const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two"); + + const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one"); + const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two"); + + const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1"); + const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2"); + const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3"); + const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4"); + const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5"); + const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6"); + const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7"); + const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8"); + const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9"); + + const auto data1 = pb.NewTuple(tupleType, {keyOne, value1}); + + const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2}); + const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3}); + + const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4}); + + const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5}); + const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6}); + const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7}); + const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8}); + const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9}); + + const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9}); + + const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideSort(pb.ExpandMap(pb.ToFlow(list), + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), + {{0U, pb.NewDataLiteral<bool>(true)}}), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(tupleType, items); } + )); + + const auto graph = setup.BuildGraph(pgmReturn); + const auto iterator = graph->GetValue().GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key one"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 1"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 3"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 2"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key one"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 4"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 9"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 8"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 7"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 6"); + UNIT_ASSERT(iterator.Next(item)); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two"); + UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 5"); + UNIT_ASSERT(!iterator.Next(item)); + UNIT_ASSERT(!iterator.Next(item)); + } +} +#endif + +} +} + diff --git a/ydb/library/yql/minikql/comp_nodes/ut/run_codegen.cmd b/ydb/library/yql/minikql/comp_nodes/ut/run_codegen.cmd new file mode 100644 index 00000000000..6dd1238df85 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/run_codegen.cmd @@ -0,0 +1,9 @@ +@echo off +call ya make -o ..\..\..\..\ && yql-minikql-comp_nodes-ut.exe +if errorlevel 1 goto :fail +echo --- DONE --- +pause +exit /b +:fail +echo --- FAIL --- +pause diff --git a/ydb/library/yql/minikql/comp_nodes/ut/ya.make b/ydb/library/yql/minikql/comp_nodes/ut/ya.make new file mode 100644 index 00000000000..2303397956f --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ut/ya.make @@ -0,0 +1,83 @@ +UNITTEST_FOR(ydb/library/yql/minikql/comp_nodes/llvm) + +FORK_SUBTESTS() + +SPLIT_FACTOR(60) + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + TIMEOUT(3600) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +SRCS( + mkql_test_factory.cpp + mkql_bit_utils_ut.cpp + mkql_block_compress_ut.cpp + mkql_block_skiptake_ut.cpp + mkql_blocks_ut.cpp + mkql_combine_ut.cpp + mkql_condense_ut.cpp + mkql_decimal_ut.cpp + mkql_chain_map_ut.cpp + mkql_chopper_ut.cpp + mkql_filters_ut.cpp + mkql_flatmap_ut.cpp + mkql_fromstring_ut.cpp + mkql_multihopping_saveload_ut.cpp + mkql_multihopping_ut.cpp + mkql_multimap_ut.cpp + mkql_fold_ut.cpp + mkql_heap_ut.cpp + mkql_compare_ut.cpp + mkql_computation_node_ut.cpp + mkql_group_ut.cpp + mkql_dict_ut.cpp + mkql_join_ut.cpp + mkql_join_dict_ut.cpp + mkql_grace_join_ut.cpp + mkql_map_join_ut.cpp + mkql_safe_circular_buffer_ut.cpp + mkql_sort_ut.cpp + mkql_switch_ut.cpp + mkql_todict_ut.cpp + mkql_variant_ut.cpp + mkql_wide_chain_map_ut.cpp + mkql_wide_chopper_ut.cpp + mkql_wide_combine_ut.cpp + mkql_wide_condense_ut.cpp + mkql_wide_filter_ut.cpp + mkql_wide_map_ut.cpp + mkql_wide_nodes_ut.cpp + mkql_wide_stream_ut.cpp + mkql_wide_top_sort_ut.cpp + mkql_listfromrange_ut.cpp + mkql_mapnext_ut.cpp + mkql_rh_hash_ut.cpp +) + +PEERDIR( + ydb/library/yql/public/udf + ydb/library/yql/public/udf/arrow + ydb/library/yql/public/udf/service/exception_policy + ydb/library/yql/sql/pg_dummy +) + +CFLAGS( + -mprfchw +) + +YQL_LAST_ABI_VERSION() + +IF (MKQL_RUNTIME_VERSION) + CFLAGS( + -DMKQL_RUNTIME_VERSION=$MKQL_RUNTIME_VERSION + ) +ENDIF() + +REQUIREMENTS(ram:10) + +END() diff --git a/ydb/library/yql/minikql/comp_nodes/ya.make b/ydb/library/yql/minikql/comp_nodes/ya.make new file mode 100644 index 00000000000..424ecfb86d0 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/ya.make @@ -0,0 +1,8 @@ +RECURSE( + llvm + no_llvm +) + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/library/yql/minikql/computation/CMakeLists.txt b/ydb/library/yql/minikql/computation/CMakeLists.txt index e519707fa41..4716de1ef37 100644 --- a/ydb/library/yql/minikql/computation/CMakeLists.txt +++ b/ydb/library/yql/minikql/computation/CMakeLists.txt @@ -7,3 +7,5 @@ add_subdirectory(llvm) +add_subdirectory(no_llvm) +add_subdirectory(ut) diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp new file mode 100644 index 00000000000..098363250a5 --- /dev/null +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp @@ -0,0 +1,124 @@ +#include <ydb/library/yql/minikql/mkql_node.h> +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_program_builder.h> +#include <ydb/library/yql/minikql/mkql_function_registry.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> +#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> +#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <vector> +#include <utility> +#include <algorithm> + +namespace NKikimr { +namespace NMiniKQL { + +namespace { +struct TSetup { + TSetup(TScopedAlloc& alloc) + : Alloc(alloc) + { + FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); + RandomProvider = CreateDeterministicRandomProvider(1); + TimeProvider = CreateDeterministicTimeProvider(10000000); + + Env.Reset(new TTypeEnvironment(Alloc)); + PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry)); + } + + THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) { + Explorer.Walk(pgm.GetNode(), *Env); + TComputationPatternOpts opts(Alloc.Ref(), *Env, GetBuiltinFactory(), + FunctionRegistry.Get(), + NUdf::EValidateMode::None, NUdf::EValidatePolicy::Exception, "OFF", EGraphPerProcess::Multi); + Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts); + TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider); + return Pattern->Clone(compOpts); + } + + TIntrusivePtr<IFunctionRegistry> FunctionRegistry; + TIntrusivePtr<IRandomProvider> RandomProvider; + TIntrusivePtr<ITimeProvider> TimeProvider; + + TScopedAlloc& Alloc; + THolder<TTypeEnvironment> Env; + THolder<TProgramBuilder> PgmBuilder; + + TExploringNodeVisitor Explorer; + IComputationPattern::TPtr Pattern; +}; +} + +Y_UNIT_TEST_SUITE(TestCompactMultiDict) { + Y_UNIT_TEST(TestIterate) { + TScopedAlloc alloc(__LOCATION__); + + TSetup setup(alloc); + + const std::vector<std::pair<ui32, std::vector<ui32>>> items = {{1, {1, 2}}, {2, {1}}, {3, {0}}, {6, {1, 7}}}; + + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + TVector<TRuntimeNode> rItems; + for (auto& [k, vv]: items) { + for (auto& v: vv) { + rItems.push_back(pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<ui32>(k), pgmBuilder.NewDataLiteral<ui32>(v)})); + } + } + auto ui32Type = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id); + auto list = pgmBuilder.NewList(pgmBuilder.NewTupleType({ui32Type, ui32Type}), rItems); + + auto dict = pgmBuilder.ToHashedDict(list, /*all*/true, + [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.Nth(item, 0); }, + [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.Nth(item, 1); }, + /*isCompact*/true, + items.size()); + + auto graph = setup.BuildGraph(dict, {}); + NUdf::TUnboxedValue res = graph->GetValue(); + + std::vector<ui32> keyVals; + for (NUdf::TUnboxedValue keys = res.GetKeysIterator(), v; keys.Next(v);) { + keyVals.push_back(v.Get<ui32>()); + } + UNIT_ASSERT_VALUES_EQUAL(keyVals.size(), items.size()); + std::sort(keyVals.begin(), keyVals.end()); + UNIT_ASSERT( + std::equal(keyVals.begin(), keyVals.end(), items.begin(), + [](ui32 l, const std::pair<ui32, std::vector<ui32>>& r) { return l == r.first; } + ) + ); + + std::vector<std::vector<ui32>> origPayloads; + for (auto& [k, vv]: items) { + origPayloads.push_back(vv); + std::sort(origPayloads.back().begin(), origPayloads.back().end()); + } + std::sort(origPayloads.begin(), origPayloads.end()); + + std::vector<std::vector<ui32>> payloadVals; + for (NUdf::TUnboxedValue payloads = res.GetPayloadsIterator(), v; payloads.Next(v);) { + payloadVals.emplace_back(); + for (NUdf::TUnboxedValue i = v.GetListIterator(), p; i.Next(p);) { + payloadVals.back().push_back(p.Get<ui32>()); + } + std::sort(payloadVals.back().begin(), payloadVals.back().end()); + } + std::sort(payloadVals.begin(), payloadVals.end()); + UNIT_ASSERT_VALUES_EQUAL(origPayloads, payloadVals); + + std::vector<std::pair<ui32, std::vector<ui32>>> vals; + for (NUdf::TUnboxedValue values = res.GetDictIterator(), k, payloads; values.NextPair(k, payloads);) { + vals.emplace_back(k.Get<ui32>(), std::vector<ui32>{}); + for (NUdf::TUnboxedValue i = payloads.GetListIterator(), p; i.Next(p);) { + vals.back().second.push_back(p.Get<ui32>()); + } + std::sort(vals.back().second.begin(), vals.back().second.end()); + } + UNIT_ASSERT_VALUES_EQUAL(items, vals); + } +} +} +} diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp index 373976d79f4..4aa1892f907 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp @@ -144,6 +144,8 @@ public: { #ifndef NDEBUG AllocState.ActiveMemInfo.emplace(MemInfo.Get(), MemInfo); +#else + Y_UNUSED(AllocState); #endif } diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp new file mode 100644 index 00000000000..5453f8b5909 --- /dev/null +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp @@ -0,0 +1,380 @@ +#include <ydb/library/yql/minikql/mkql_node.h> +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_program_builder.h> +#include <ydb/library/yql/minikql/mkql_function_registry.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h> +#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> +#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h> + +#include <library/cpp/testing/unittest/registar.h> + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + TIntrusivePtr<IRandomProvider> CreateRandomProvider() { + return CreateDeterministicRandomProvider(1); + } + + TIntrusivePtr<ITimeProvider> CreateTimeProvider() { + return CreateDeterministicTimeProvider(10000000); + } + + TComputationNodeFactory GetAuxCallableFactory() { + return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* { + if (callable.GetType()->GetName() == "OneYieldStream") { + return new TExternalComputationNode(ctx.Mutables); + } + + return GetBuiltinFactory()(callable, ctx); + }; + } + + struct TSetup { + TSetup(TScopedAlloc& alloc) + : Alloc(alloc) + { + FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); + RandomProvider = CreateRandomProvider(); + TimeProvider = CreateTimeProvider(); + + Env.Reset(new TTypeEnvironment(Alloc)); + PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry)); + } + + THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) { + Explorer.Walk(pgm.GetNode(), *Env); + TComputationPatternOpts opts(Alloc.Ref(), *Env, GetAuxCallableFactory(), + FunctionRegistry.Get(), + NUdf::EValidateMode::None, NUdf::EValidatePolicy::Fail, "OFF", EGraphPerProcess::Multi); + Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts); + TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider); + return Pattern->Clone(compOpts); + } + + TIntrusivePtr<IFunctionRegistry> FunctionRegistry; + TIntrusivePtr<IRandomProvider> RandomProvider; + TIntrusivePtr<ITimeProvider> TimeProvider; + + TScopedAlloc& Alloc; + THolder<TTypeEnvironment> Env; + THolder<TProgramBuilder> PgmBuilder; + + TExploringNodeVisitor Explorer; + IComputationPattern::TPtr Pattern; + }; + + struct TStreamWithYield : public NUdf::TBoxedValue { + TStreamWithYield(const TUnboxedValueVector& items, ui32 yieldPos, ui32 index) + : Items(items) + , YieldPos(yieldPos) + , Index(index) + {} + + private: + TUnboxedValueVector Items; + ui32 YieldPos; + ui32 Index; + + ui32 GetTraverseCount() const override { + return 0; + } + + NUdf::TUnboxedValue Save() const override { + return NUdf::TUnboxedValue::Zero(); + } + + void Load(const NUdf::TStringRef& state) override { + Y_UNUSED(state); + } + + NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) final { + if (Index >= Items.size()) { + return NUdf::EFetchStatus::Finish; + } + if (Index == YieldPos) { + return NUdf::EFetchStatus::Yield; + } + result = Items[Index++]; + return NUdf::EFetchStatus::Ok; + } + }; +} + +Y_UNIT_TEST_SUITE(TMiniKQLSaveLoadTest) { + Y_UNIT_TEST(TestSqueezeSaveLoad) { + TScopedAlloc alloc(__LOCATION__); + + const std::vector<ui32> items = {2, 3, 4, 5, 6, 7, 8}; + + auto buildGraph = [&items] (TSetup& setup, ui32 yieldPos, ui32 startIndex) -> THolder<IComputationGraph> { + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto dataType = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id); + auto streamType = pgmBuilder.NewStreamType(dataType); + + TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", streamType); + auto streamNode = inStream.Build(); + + auto pgmReturn = pgmBuilder.Squeeze( + TRuntimeNode(streamNode, false), + pgmBuilder.NewDataLiteral<ui32>(1), + [&](TRuntimeNode item, TRuntimeNode state) { + return pgmBuilder.Add(item, state); + }, + [](TRuntimeNode state) { + return state; + }, + [](TRuntimeNode state) { + return state; + }); + + TUnboxedValueVector streamItems; + for (auto item : items) { + streamItems.push_back(NUdf::TUnboxedValuePod(item)); + } + + auto graph = setup.BuildGraph(pgmReturn, {streamNode}); + auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex)); + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue)); + return graph; + }; + + for (ui32 yieldPos = 0; yieldPos < items.size(); ++yieldPos) { + TSetup setup1(alloc); + auto graph1 = buildGraph(setup1, yieldPos, 0); + + auto root1 = graph1->GetValue(); + NUdf::TUnboxedValue res; + auto status = root1.Fetch(res); + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield); + + TString graphState; + SaveGraphState(&root1, 1, 0ULL, graphState); + + TSetup setup2(alloc); + auto graph2 = buildGraph(setup2, -1, yieldPos); + + auto root2 = graph2->GetValue(); + LoadGraphState(&root2, 1, 0ULL, graphState); + + status = root2.Fetch(res); + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Ok); + UNIT_ASSERT_EQUAL(res.Get<ui32>(), 36); + + status = root2.Fetch(res); + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish); + } + } + + Y_UNIT_TEST(TestSqueeze1SaveLoad) { + TScopedAlloc alloc(__LOCATION__); + + const std::vector<ui32> items = {1, 2, 3, 4, 5, 6, 7, 8}; + + auto buildGraph = [&items] (TSetup& setup, ui32 yieldPos, ui32 startIndex) -> THolder<IComputationGraph> { + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto dataType = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id); + auto streamType = pgmBuilder.NewStreamType(dataType); + + TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", streamType); + auto streamNode = inStream.Build(); + + auto pgmReturn = pgmBuilder.Squeeze1( + TRuntimeNode(streamNode, false), + [](TRuntimeNode item) { + return item; + }, + [&](TRuntimeNode item, TRuntimeNode state) { + return pgmBuilder.Add(item, state); + }, + [](TRuntimeNode state) { + return state; + }, + [](TRuntimeNode state) { + return state; + }); + + TUnboxedValueVector streamItems; + for (auto item : items) { + streamItems.push_back(NUdf::TUnboxedValuePod(item)); + } + + auto graph = setup.BuildGraph(pgmReturn, {streamNode}); + auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex)); + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue)); + return graph; + }; + + for (ui32 yieldPos = 0; yieldPos < items.size(); ++yieldPos) { + TSetup setup1(alloc); + auto graph1 = buildGraph(setup1, yieldPos, 0); + + auto root1 = graph1->GetValue(); + + NUdf::TUnboxedValue res; + auto status = root1.Fetch(res); + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield); + + TString graphState; + SaveGraphState(&root1, 1, 0ULL, graphState); + + TSetup setup2(alloc); + auto graph2 = buildGraph(setup2, -1, yieldPos); + + auto root2 = graph2->GetValue(); + LoadGraphState(&root2, 1, 0ULL, graphState); + + status = root2.Fetch(res); + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Ok); + UNIT_ASSERT_EQUAL(res.Get<ui32>(), 36); + + status = root2.Fetch(res); + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish); + } + } + + Y_UNIT_TEST(TestHoppingSaveLoad) { + TScopedAlloc alloc(__LOCATION__); + + const std::vector<std::pair<i64, ui32>> items = { + {1, 2}, + {2, 3}, + {15, 4}, + {23, 6}, + {24, 5}, + {25, 7}, + {40, 2}, + {47, 1}, + {51, 6}, + {59, 2}, + {85, 8}, + {55, 1000}, + {200, 0} + }; + + auto buildGraph = [&items] (TSetup& setup, ui32 yieldPos, ui32 startIndex) -> THolder<IComputationGraph> { + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto structType = pgmBuilder.NewEmptyStructType(); + structType = pgmBuilder.NewStructType(structType, "time", + pgmBuilder.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id)); + structType = pgmBuilder.NewStructType(structType, "sum", + pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); + auto timeIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("time"); + auto sumIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("sum"); + + auto inStreamType = pgmBuilder.NewStreamType(structType); + + TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", inStreamType); + auto streamNode = inStream.Build(); + + ui64 hop = 10, interval = 30, delay = 20; + + auto pgmReturn = pgmBuilder.HoppingCore( + TRuntimeNode(streamNode, false), + [&](TRuntimeNode item) { // timeExtractor + return pgmBuilder.Member(item, "time"); + }, + [&](TRuntimeNode item) { // init + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("sum", pgmBuilder.Member(item, "sum")); + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode item, TRuntimeNode state) { // update + auto add = pgmBuilder.AggrAdd( + pgmBuilder.Member(item, "sum"), + pgmBuilder.Member(state, "sum")); + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("sum", add); + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode state) { // save + return pgmBuilder.Member(state, "sum"); + }, + [&](TRuntimeNode savedState) { // load + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("sum", savedState); + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode state1, TRuntimeNode state2) { // merge + auto add = pgmBuilder.AggrAdd( + pgmBuilder.Member(state1, "sum"), + pgmBuilder.Member(state2, "sum")); + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("sum", add); + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode state, TRuntimeNode time) { // finish + Y_UNUSED(time); + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("sum", pgmBuilder.Member(state, "sum")); + return pgmBuilder.NewStruct(members); + }, + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&hop, sizeof(hop))), // hop + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&interval, sizeof(interval))), // interval + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&delay, sizeof(delay))) // delay + ); + + auto graph = setup.BuildGraph(pgmReturn, {streamNode}); + + TUnboxedValueVector streamItems; + for (size_t i = 0; i < items.size(); ++i) { + NUdf::TUnboxedValue* itemsPtr; + auto structValues = graph->GetHolderFactory().CreateDirectArrayHolder(2, itemsPtr); + itemsPtr[timeIndex] = NUdf::TUnboxedValuePod(items[i].first); + itemsPtr[sumIndex] = NUdf::TUnboxedValuePod(items[i].second); + streamItems.push_back(std::move(structValues)); + } + + auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex)); + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue)); + return graph; + }; + + for (ui32 yieldPos = 0; yieldPos < items.size(); ++yieldPos) { + std::vector<ui32> result; + + TSetup setup1(alloc); + auto graph1 = buildGraph(setup1, yieldPos, 0); + auto root1 = graph1->GetValue(); + + NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok; + while (status == NUdf::EFetchStatus::Ok) { + NUdf::TUnboxedValue val; + status = root1.Fetch(val); + if (status == NUdf::EFetchStatus::Ok) { + result.push_back(val.GetElement(0).Get<ui32>()); + } + } + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield); + + TString graphState; + SaveGraphState(&root1, 1, 0ULL, graphState); + + TSetup setup2(alloc); + auto graph2 = buildGraph(setup2, -1, yieldPos); + auto root2 = graph2->GetValue(); + LoadGraphState(&root2, 1, 0ULL, graphState); + + status = NUdf::EFetchStatus::Ok; + while (status == NUdf::EFetchStatus::Ok) { + NUdf::TUnboxedValue val; + status = root2.Fetch(val); + if (status == NUdf::EFetchStatus::Ok) { + result.push_back(val.GetElement(0).Get<ui32>()); + } + } + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish); + + const std::vector<ui32> resultCompare = {5, 9, 27, 22, 21, 11, 11, 8, 8, 8, 8}; + UNIT_ASSERT_EQUAL(result, resultCompare); + } + } +} + +} // namespace NMiniKQL +} // namespace NKikimr diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_list_ut.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_list_ut.cpp new file mode 100644 index 00000000000..a07f1b05c7f --- /dev/null +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_list_ut.cpp @@ -0,0 +1,126 @@ +#include "mkql_computation_node_list.h" + +#include <ydb/library/yql/minikql/mkql_function_registry.h> +#include <library/cpp/testing/unittest/registar.h> + +namespace NKikimr { +namespace NMiniKQL { +Y_UNIT_TEST_SUITE(TestListRepresentation) { + Y_UNIT_TEST(Test) { + TMemoryUsageInfo memInfo(TStringBuf("test")); + TScopedAlloc alloc(__LOCATION__); + using TListType = TListRepresentation<ui32, 256>; + TListType list1; + auto it1 = list1.GetIterator(); + UNIT_ASSERT(it1.AtEnd()); + + TListType list2(list1, 0); + auto it2 = list2.GetIterator(); + UNIT_ASSERT(!it2.AtEnd()); + UNIT_ASSERT_VALUES_EQUAL(it2.Current(), 0); + it2.Next(); + UNIT_ASSERT(it2.AtEnd()); + + TVector<TListType> lists; + lists.push_back(list1); + const ui32 n = 100; + for (ui32 i = 0; i < n; ++i) { + auto c = i; + lists.push_back(TListType(lists.back(), std::move(c))); + } + + for (ui32 i = 0; i < n; ++i) { + auto list = lists[i]; + auto it = list.GetIterator(); + ui32 expected = 0; + while (!it.AtEnd()) { + UNIT_ASSERT_VALUES_EQUAL(it.Current(), expected); + ++expected; + it.Next(); + } + + UNIT_ASSERT_VALUES_EQUAL(expected, i); + } + + for (ui32 i = 0; i < n; ++i) { + auto list = lists[i]; + auto it = list.GetReverseIterator(); + ui32 expected = i; + while (!it.AtEnd()) { + UNIT_ASSERT_VALUES_EQUAL(it.Current(), --expected); + it.Next(); + } + + UNIT_ASSERT_VALUES_EQUAL(expected, 0); + } + + TVector<TListType> rlists; + rlists.push_back(list1); + for (ui32 i = 0; i < n; ++i) { + auto c = i; + rlists.push_back(TListType(std::move(c), rlists.back())); + } + + for (ui32 i = 0; i < n; ++i) { + auto list = rlists[i]; + auto it = list.GetIterator(); + ui32 expected = i; + while (!it.AtEnd()) { + UNIT_ASSERT_VALUES_EQUAL(it.Current(), --expected); + it.Next(); + } + + UNIT_ASSERT_VALUES_EQUAL(expected, 0); + } + + for (ui32 i = 0; i < n; ++i) { + auto list = rlists[i]; + auto it = list.GetReverseIterator(); + ui32 expected = 0; + while (!it.AtEnd()) { + UNIT_ASSERT_VALUES_EQUAL(it.Current(), expected); + ++expected; + it.Next(); + } + + UNIT_ASSERT_VALUES_EQUAL(expected, i); + } + + TVector<TListType> zlists; + zlists.push_back(list1); + for (ui32 i = 0; i < n; ++i) { + zlists.push_back(zlists.back().Append(2 * i)); + zlists.push_back(zlists.back().Prepend(2 * i + 1)); + } + + for (ui32 i = 0; i < 2 * n; ++i) { + ui32 k = (i + 1) / 2; + auto list = zlists[i]; + auto it = list.GetIterator(); + ui32 expected = 2 * k + 1; + if ((i % 2)) + expected -= 2; + + if (i >= 2) { + while (!it.AtEnd()) { + expected -= 2; + UNIT_ASSERT_VALUES_EQUAL(it.Current(), expected); + it.Next(); + if (expected == 1) + break; + } + } + + expected = 0; + while (!it.AtEnd()) { + UNIT_ASSERT_VALUES_EQUAL(it.Current(), expected); + expected += 2; + it.Next(); + } + + UNIT_ASSERT_VALUES_EQUAL(expected, 2 * k); + } + } +} +} +} diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp new file mode 100644 index 00000000000..6234ef827bb --- /dev/null +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp @@ -0,0 +1,863 @@ +#include "mkql_computation_node_pack.h" +#include "mkql_computation_node_holders.h" +#include "mkql_block_builder.h" +#include "mkql_block_reader.h" +#include <ydb/library/yql/minikql/mkql_function_registry.h> +#include <ydb/library/yql/minikql/mkql_node.h> +#include <ydb/library/yql/minikql/mkql_program_builder.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> +#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> +#include <ydb/library/yql/public/udf/arrow/util.h> + +#include <library/cpp/random_provider/random_provider.h> +#include <ydb/library/yql/minikql/aligned_page_pool.h> + +#include <ydb/library/yql/public/udf/udf_value.h> + +#include <library/cpp/testing/unittest/registar.h> +#include <util/generic/ylimits.h> +#include <util/generic/xrange.h> +#include <util/generic/maybe.h> +#include <util/string/cast.h> +#include <util/string/builder.h> +#include <util/system/hp_timer.h> +#include <util/system/sanitizers.h> + +namespace NKikimr { +namespace NMiniKQL { + +#ifdef WITH_VALGRIND +constexpr static size_t PERFORMANCE_COUNT = 0x1000; +#elifdef NDEBUG +constexpr static size_t PERFORMANCE_COUNT = NSan::PlainOrUnderSanitizer(0x4000000, 0x1000); +#else +constexpr static size_t PERFORMANCE_COUNT = NSan::PlainOrUnderSanitizer(0x1000000, 0x1000); +#endif + +template<bool Fast, bool Transport> +struct TPackerTraits; + +template<bool Fast> +struct TPackerTraits<Fast, false> { + using TPackerType = TValuePackerGeneric<Fast>; +}; + +template<bool Fast> +struct TPackerTraits<Fast, true> { + using TPackerType = TValuePackerTransport<Fast>; +}; + +using NDetails::TChunkedInputBuffer; + +template<bool Fast, bool Transport> +class TMiniKQLComputationNodePackTest: public TTestBase { + using TValuePackerType = typename TPackerTraits<Fast, Transport>::TPackerType; +protected: + TMiniKQLComputationNodePackTest() + : FunctionRegistry(CreateFunctionRegistry(CreateBuiltinRegistry())) + , RandomProvider(CreateDefaultRandomProvider()) + , Alloc(__LOCATION__) + , Env(Alloc) + , PgmBuilder(Env, *FunctionRegistry) + , MemInfo("Memory") + , HolderFactory(Alloc.Ref(), MemInfo, FunctionRegistry.Get()) + , ArrowPool_(arrow::default_memory_pool()) + { + } + + void TestNumericTypes() { + TestNumericType<bool>(NUdf::TDataType<bool>::Id); + TestNumericType<ui8>(NUdf::TDataType<ui8>::Id); + TestNumericType<i32>(NUdf::TDataType<i32>::Id); + TestNumericType<i64>(NUdf::TDataType<i64>::Id); + TestNumericType<ui32>(NUdf::TDataType<ui32>::Id); + TestNumericType<ui64>(NUdf::TDataType<ui64>::Id); + TestNumericType<float>(NUdf::TDataType<float>::Id); + TestNumericType<double>(NUdf::TDataType<double>::Id); + } + + void TestOptionalNumericTypes() { + TestOptionalNumericType<bool>(NUdf::TDataType<bool>::Id); + TestOptionalNumericType<ui8>(NUdf::TDataType<ui8>::Id); + TestOptionalNumericType<i32>(NUdf::TDataType<i32>::Id); + TestOptionalNumericType<i64>(NUdf::TDataType<i64>::Id); + TestOptionalNumericType<ui32>(NUdf::TDataType<ui32>::Id); + TestOptionalNumericType<ui64>(NUdf::TDataType<ui64>::Id); + TestOptionalNumericType<float>(NUdf::TDataType<float>::Id); + TestOptionalNumericType<double>(NUdf::TDataType<double>::Id); + } + + void TestStringTypes() { + TestStringType(NUdf::TDataType<NUdf::TUtf8>::Id); + TestStringType(NUdf::TDataType<char*>::Id); + TestStringType(NUdf::TDataType<NUdf::TYson>::Id); + TestStringType(NUdf::TDataType<NUdf::TJson>::Id); + } + + void TestOptionalStringTypes() { + TestOptionalStringType(NUdf::TDataType<NUdf::TUtf8>::Id); + TestOptionalStringType(NUdf::TDataType<char*>::Id); + TestOptionalStringType(NUdf::TDataType<NUdf::TYson>::Id); + TestOptionalStringType(NUdf::TDataType<NUdf::TJson>::Id); + } + + void TestListType() { + TDefaultListRepresentation listValues; + for (ui32 val: xrange(0, 10)) { + listValues = listValues.Append(NUdf::TUnboxedValuePod(val)); + } + TType* listType = PgmBuilder.NewListType(PgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); + const NUdf::TUnboxedValue value = HolderFactory.CreateDirectListHolder(std::move(listValues)); + const auto uValue = TestPackUnpack(listType, value, "Type:List(ui32)"); + + UNIT_ASSERT_VALUES_EQUAL(uValue.GetListLength(), 10); + ui32 i = 0; + const auto iter = uValue.GetListIterator(); + for (NUdf::TUnboxedValue item; iter.Next(item); ++i) { + UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), i); + } + UNIT_ASSERT_VALUES_EQUAL(i, 10); + } + + void TestListOfOptionalsType() { + TDefaultListRepresentation listValues; + for (ui32 i: xrange(0, 10)) { + NUdf::TUnboxedValue uVal = NUdf::TUnboxedValuePod(); + if (i % 2) { + uVal = MakeString(TString(i * 2, '0' + i)); + } + listValues = listValues.Append(std::move(uVal)); + } + TType* listType = PgmBuilder.NewListType(PgmBuilder.NewOptionalType(PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id))); + const NUdf::TUnboxedValue value = HolderFactory.CreateDirectListHolder(std::move(listValues)); + const auto uValue = TestPackUnpack(listType, value, "Type:List(Optional(utf8))"); + + UNIT_ASSERT_VALUES_EQUAL(uValue.GetListLength(), 10); + ui32 i = 0; + const auto iter = uValue.GetListIterator(); + for (NUdf::TUnboxedValue uVal; iter.Next(uVal); ++i) { + if (i % 2) { + UNIT_ASSERT(uVal); + UNIT_ASSERT_VALUES_EQUAL(std::string_view(uVal.AsStringRef()), TString(i * 2, '0' + i)); + } else { + UNIT_ASSERT(!uVal); + } + } + UNIT_ASSERT_VALUES_EQUAL(i, 10); + } + + void TestTupleType() { + std::vector<TType*> tupleElemenTypes; + tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)); + tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0])); + tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0])); + tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)); + tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[3])); + tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[3])); + TType* tupleType = PgmBuilder.NewTupleType(tupleElemenTypes); + + TUnboxedValueVector tupleElemens; + tupleElemens.push_back(MakeString("01234567890123456789")); + tupleElemens.push_back(MakeString("01234567890")); + tupleElemens.push_back(NUdf::TUnboxedValuePod()); + tupleElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345))); + tupleElemens.push_back(NUdf::TUnboxedValuePod()); + tupleElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345))); + + const NUdf::TUnboxedValue value = HolderFactory.VectorAsArray(tupleElemens); + const auto uValue = TestPackUnpack(tupleType, value, "Type:Tuple"); + + { + auto e = uValue.GetElement(0); + UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890123456789"); + } + { + auto e = uValue.GetElement(1); + UNIT_ASSERT(e); + UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890"); + } + { + auto e = uValue.GetElement(2); + UNIT_ASSERT(!e); + } + { + auto e = uValue.GetElement(3); + UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345); + } + { + auto e = uValue.GetElement(4); + UNIT_ASSERT(!e); + } + { + auto e = uValue.GetElement(5); + UNIT_ASSERT(e); + UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345); + } + } + + void TestStructType() { + const std::vector<std::pair<std::string_view, TType*>> structElemenTypes = { + {"a", PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)}, + {"b", PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id, true)}, + {"c", PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id, true)}, + {"d", PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)}, + {"e", PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id, true)}, + {"f", PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id, true)} + }; + TType* structType = PgmBuilder.NewStructType(structElemenTypes); + + TUnboxedValueVector structElemens; + structElemens.push_back(MakeString("01234567890123456789")); + structElemens.push_back(MakeString("01234567890")); + structElemens.push_back(NUdf::TUnboxedValuePod()); + structElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345))); + structElemens.push_back(NUdf::TUnboxedValuePod()); + structElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345))); + + const NUdf::TUnboxedValue value = HolderFactory.VectorAsArray(structElemens); + const auto uValue = TestPackUnpack(structType, value, "Type:Struct"); + + { + auto e = uValue.GetElement(0); + UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890123456789"); + } + { + auto e = uValue.GetElement(1); + UNIT_ASSERT(e); + UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890"); + } + { + auto e = uValue.GetElement(2); + UNIT_ASSERT(!e); + } + { + auto e = uValue.GetElement(3); + UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345); + } + { + auto e = uValue.GetElement(4); + UNIT_ASSERT(!e); + } + { + auto e = uValue.GetElement(5); + UNIT_ASSERT(e); + UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345); + } + } + + void TestOptionalType() { + TType* type = PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id); + type = PgmBuilder.NewOptionalType(type); + type = PgmBuilder.NewOptionalType(type); + type = PgmBuilder.NewOptionalType(type); + + NUdf::TUnboxedValue uValue = NUdf::TUnboxedValuePod(); + uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:null"); + UNIT_ASSERT(!uValue); + + uValue = NUdf::TUnboxedValuePod(ui64(123)); + uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:opt(opt(opt(123)))"); + UNIT_ASSERT_VALUES_EQUAL(uValue.Get<ui64>(), 123); + + uValue = NUdf::TUnboxedValuePod().MakeOptional().MakeOptional(); + uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:opt(opt(null))"); + UNIT_ASSERT(uValue); + uValue = uValue.GetOptionalValue(); + UNIT_ASSERT(uValue); + uValue = uValue.GetOptionalValue(); + UNIT_ASSERT(!uValue); + + uValue = NUdf::TUnboxedValuePod().MakeOptional(); + uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:opt(null)"); + UNIT_ASSERT(uValue); + uValue = uValue.GetOptionalValue(); + UNIT_ASSERT(!uValue); + } + + void TestDictType() { + TType* keyType = PgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id); + TType* payloadType = PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id); + TType* dictType = PgmBuilder.NewDictType(keyType, payloadType, false); + + TValuesDictHashSingleFixedMap<ui32> map; + map[4] = NUdf::TUnboxedValuePod::Embedded("4"); + map[10] = NUdf::TUnboxedValuePod::Embedded("10"); + map[1] = NUdf::TUnboxedValuePod::Embedded("1"); + const NUdf::TUnboxedValue value = HolderFactory.CreateDirectHashedSingleFixedMapHolder<ui32, false>(std::move(map), std::nullopt); + + const auto uValue = TestPackUnpack(dictType, value, "Type:Dict"); + + UNIT_ASSERT_VALUES_EQUAL(uValue.GetDictLength(), 3); + const auto it = uValue.GetDictIterator(); + for (NUdf::TUnboxedValue key, payload; it.NextPair(key, payload);) { + UNIT_ASSERT_VALUES_EQUAL(ToString(key.template Get<ui32>()), std::string_view(payload.AsStringRef())); + } + } + + void TestVariantTypeOverStruct() { + const std::vector<std::pair<std::string_view, TType*>> structElemenTypes = { + {"a", PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)}, + {"b", PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id, true)}, + {"d", PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)} + }; + TType* structType = PgmBuilder.NewStructType(structElemenTypes); + TestVariantTypeImpl(PgmBuilder.NewVariantType(structType)); + } + + void TestVariantTypeOverTuple() { + std::vector<TType*> tupleElemenTypes; + tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)); + tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0])); + tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)); + TType* tupleType = PgmBuilder.NewTupleType(tupleElemenTypes); + TestVariantTypeImpl(PgmBuilder.NewVariantType(tupleType)); + } + + void ValidateEmbeddedLength(TRope buf, const TString& info) { + size_t size = buf.GetSize(); + TChunkedInputBuffer chunked(std::move(buf)); + return ValidateEmbeddedLength(chunked, size, info); + } + + void ValidateEmbeddedLength(TStringBuf buf, const TString& info) { + TChunkedInputBuffer chunked(buf); + return ValidateEmbeddedLength(chunked, buf.size(), info); + } + + void ValidateEmbeddedLength(TChunkedInputBuffer& buf, size_t totalSize, const TString& info) { + if constexpr (!Fast) { + if (totalSize > 8) { + ui32 len = NDetails::GetRawData<ui32>(buf); + UNIT_ASSERT_VALUES_EQUAL_C(len + 4, totalSize, info); + } else { + ui32 len = NDetails::GetRawData<ui8>(buf); + UNIT_ASSERT_VALUES_EQUAL_C(((len & 0x0f) >> 1) + 1, totalSize, info); + } + } + } + + void TestPackPerformance(TType* type, const NUdf::TUnboxedValuePod& uValue) + { + TValuePackerType packer(false, type); + const THPTimer timer; + for (size_t i = 0U; i < PERFORMANCE_COUNT; ++i) + packer.Pack(uValue); + Cerr << timer.Passed() << Endl; + } + + NUdf::TUnboxedValue TestPackUnpack(TValuePackerType& packer, const NUdf::TUnboxedValuePod& uValue, + const TString& additionalMsg, const std::optional<ui32>& expectedLength = {}) + { + auto packedValue = packer.Pack(uValue); + if constexpr (Transport) { + if (expectedLength) { + UNIT_ASSERT_VALUES_EQUAL_C(packedValue.size(), *expectedLength, additionalMsg); + } + ValidateEmbeddedLength(packedValue, additionalMsg); + return packer.Unpack(std::move(packedValue), HolderFactory); + } else { + if (expectedLength) { + UNIT_ASSERT_VALUES_EQUAL_C(packedValue.Size(), *expectedLength, additionalMsg); + } + ValidateEmbeddedLength(packedValue, additionalMsg); + return packer.Unpack(packedValue, HolderFactory); + } + } + + NUdf::TUnboxedValue TestPackUnpack(TType* type, const NUdf::TUnboxedValuePod& uValue, const TString& additionalMsg, + const std::optional<ui32>& expectedLength = {}) + { + TValuePackerType packer(false, type); + return TestPackUnpack(packer, uValue, additionalMsg, expectedLength); + } + + template <typename T> + void TestNumericValue(T value, TValuePackerType& packer, const TString& typeDesc) { + TString additionalMsg = TStringBuilder() << typeDesc << ", Value:" << value; + auto uValue = TestPackUnpack(packer, NUdf::TUnboxedValuePod(value), additionalMsg); + UNIT_ASSERT_VALUES_EQUAL_C(uValue.template Get<T>(), value, additionalMsg); + } + + template <typename T> + void TestNumericType(NUdf::TDataTypeId schemeType) { + TString typeDesc = TStringBuilder() << ", Type:" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name; + TValuePackerType packer(false, PgmBuilder.NewDataType(schemeType)); + + TestNumericValue<T>(Max<T>(), packer, typeDesc); + TestNumericValue<T>(Min<T>(), packer, typeDesc); + TestNumericValue<T>(T(0), packer, typeDesc); + TestNumericValue<T>(T(1), packer, typeDesc); + } + + template <typename T> + void TestOptionalNumericValue(std::optional<T> value, TValuePackerType& packer, const TString& typeDesc, + const std::optional<ui32>& expectedLength = {}) + { + TString additionalMsg = TStringBuilder() << typeDesc << "), Value:" << (value ? ToString(*value) : TString("null")); + const auto v = value ? NUdf::TUnboxedValuePod(*value) : NUdf::TUnboxedValuePod(); + const auto uValue = TestPackUnpack(packer, v, additionalMsg, expectedLength); + if (value) { + UNIT_ASSERT_VALUES_EQUAL_C(uValue.template Get<T>(), *value, additionalMsg); + } else { + UNIT_ASSERT_C(!uValue, additionalMsg); + } + } + + template <typename T> + void TestOptionalNumericType(NUdf::TDataTypeId schemeType) { + TString typeDesc = TStringBuilder() << ", Type:Optional(" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name; + TValuePackerType packer(false, PgmBuilder.NewOptionalType(PgmBuilder.NewDataType(schemeType))); + TestOptionalNumericValue<T>(std::optional<T>(Max<T>()), packer, typeDesc); + TestOptionalNumericValue<T>(std::optional<T>(Min<T>()), packer, typeDesc); + TestOptionalNumericValue<T>(std::optional<T>(), packer, typeDesc, 1); + TestOptionalNumericValue<T>(std::optional<T>(0), packer, typeDesc); + TestOptionalNumericValue<T>(std::optional<T>(1), packer, typeDesc); + } + + void TestStringValue(const std::string_view& value, TValuePackerType& packer, const TString& typeDesc, ui32 expectedLength) { + TString additionalMsg = TStringBuilder() << typeDesc << ", Value:" << value; + const auto v = NUdf::TUnboxedValue(MakeString(value)); + const auto uValue = TestPackUnpack(packer, v, additionalMsg, expectedLength); + UNIT_ASSERT_VALUES_EQUAL_C(std::string_view(uValue.AsStringRef()), value, additionalMsg); + } + + void TestStringType(NUdf::TDataTypeId schemeType) { + TString typeDesc = TStringBuilder() << ", Type:" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name; + TValuePackerType packer(false, PgmBuilder.NewDataType(schemeType)); + TestStringValue("0123456789012345678901234567890123456789", packer, typeDesc, 40 + 4); + TestStringValue("[]", packer, typeDesc, Fast ? (2 + 4) : (2 + 1)); + TestStringValue("1234567", packer, typeDesc, Fast ? (7 + 4) : (7 + 1)); + TestStringValue("", packer, typeDesc, Fast ? (0 + 4) : (0 + 1)); + TestStringValue("12345678", packer, typeDesc, 8 + 4); + + TString hugeString(12345678, 'X'); + TestStringValue(hugeString, packer, typeDesc, hugeString.size() + 4); + } + + void TestUuidType() { + auto schemeType = NUdf::TDataType<NUdf::TUuid>::Id; + TString typeDesc = TStringBuilder() << ", Type:" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name; + TValuePackerType packer(false, PgmBuilder.NewDataType(schemeType)); + TestStringValue("0123456789abcdef", packer, typeDesc, Fast ? 16 : (16 + 4)); + } + + void TestOptionalStringValue(std::optional<std::string_view> value, TValuePackerType& packer, const TString& typeDesc, ui32 expectedLength) { + TString additionalMsg = TStringBuilder() << typeDesc << "), Value:" << (value ? *value : TString("null")); + const auto v = value ? NUdf::TUnboxedValue(MakeString(*value)) : NUdf::TUnboxedValue(); + const auto uValue = TestPackUnpack(packer, v, additionalMsg, expectedLength); + if (value) { + UNIT_ASSERT_VALUES_EQUAL_C(std::string_view(uValue.AsStringRef()), *value, additionalMsg); + } else { + UNIT_ASSERT_C(!uValue, additionalMsg); + } + } + + void TestOptionalStringType(NUdf::TDataTypeId schemeType) { + TString typeDesc = TStringBuilder() << ", Type:Optional(" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name; + TValuePackerType packer(false, PgmBuilder.NewOptionalType(PgmBuilder.NewDataType(schemeType))); + TestOptionalStringValue("0123456789012345678901234567890123456789", packer, typeDesc, Fast ? (40 + 4 + 1) : (40 + 4)); + TestOptionalStringValue(std::nullopt, packer, typeDesc, 1); + TestOptionalStringValue("[]", packer, typeDesc, Fast ? (2 + 4 + 1) : (2 + 1)); + TestOptionalStringValue("1234567", packer, typeDesc, Fast ? (7 + 4 + 1) : (7 + 1)); + TestOptionalStringValue("", packer, typeDesc, Fast ? (0 + 4 + 1) : 1); + TestOptionalStringValue("12345678", packer, typeDesc, Fast ? (8 + 4 + 1) : (8 + 4)); + } + + void TestVariantTypeImpl(TType* variantType) { + TString descr = TStringBuilder() << "Type:Variant(" + << static_cast<TVariantType*>(variantType)->GetUnderlyingType()->GetKindAsStr() << ')'; + { + const NUdf::TUnboxedValue value = HolderFactory.CreateVariantHolder(MakeString("01234567890123456789"), 0); + const auto uValue = TestPackUnpack(variantType, value, descr); + + UNIT_ASSERT_VALUES_EQUAL(uValue.GetVariantIndex(), 0); + auto e = uValue.GetVariantItem(); + UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890123456789"); + } + { + const NUdf::TUnboxedValue value = HolderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(), 1); + const auto uValue = TestPackUnpack(variantType, value, descr); + + UNIT_ASSERT_VALUES_EQUAL(uValue.GetVariantIndex(), 1); + auto e = uValue.GetVariantItem(); + UNIT_ASSERT(!e); + } + { + const NUdf::TUnboxedValue value = HolderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(ui64(12345)), 2); + const auto uValue = TestPackUnpack(variantType, value, descr); + + UNIT_ASSERT_VALUES_EQUAL(uValue.GetVariantIndex(), 2); + auto e = uValue.GetVariantItem(); + UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345ull); + } + } + + NUdf::TUnboxedValue MakeTupleValue(TType*& tupleType) { + std::vector<TType*> tupleElemenTypes; + tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)); + tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0])); + tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0])); + tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)); + tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[3])); + tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[3])); + tupleType = PgmBuilder.NewTupleType(tupleElemenTypes); + + TUnboxedValueVector tupleElemens; + tupleElemens.push_back(MakeString("01234567890123456789")); + tupleElemens.push_back(MakeString("01234567890")); + tupleElemens.push_back(NUdf::TUnboxedValuePod()); + tupleElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345))); + tupleElemens.push_back(NUdf::TUnboxedValuePod()); + tupleElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345))); + + return HolderFactory.VectorAsArray(tupleElemens); + } + + void ValidateTupleValue(const NUdf::TUnboxedValue& value) { + using NYql::NUdf::TStringValue; + UNIT_ASSERT(value.IsBoxed()); + + auto e0 = value.GetElement(0); + auto e1 = value.GetElement(1); + UNIT_ASSERT_VALUES_EQUAL(std::string_view(e0.AsStringRef()), "01234567890123456789"); + UNIT_ASSERT_VALUES_EQUAL(std::string_view(e1.AsStringRef()), "01234567890"); + UNIT_ASSERT(!value.GetElement(2).HasValue()); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(3).Get<ui64>(), 12345); + UNIT_ASSERT(!value.GetElement(4).HasValue()); + UNIT_ASSERT_VALUES_EQUAL(value.GetElement(5).Get<ui64>(), 12345); + } + + void TestTuplePackPerformance() { + TType* tupleType; + const auto value = MakeTupleValue(tupleType); + TestPackPerformance(tupleType, value); + } + + void TestPairPackPerformance() { + std::vector<TType*> tupleElemenTypes; + tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); + tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); + TType* tupleType = PgmBuilder.NewTupleType(tupleElemenTypes); + + TUnboxedValueVector tupleElemens; + tupleElemens.push_back(NUdf::TUnboxedValuePod(ui32(12345))); + tupleElemens.push_back(NUdf::TUnboxedValuePod(ui32(67890))); + + const NUdf::TUnboxedValue value = HolderFactory.VectorAsArray(tupleElemens); + TestPackPerformance(tupleType, value); + } + + void TestShortStringPackPerformance() { + const auto v = NUdf::TUnboxedValuePod::Embedded("01234"); + TType* type = PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id); + TestPackPerformance(type, v); + } + + void TestIntegerPackPerformance() { + const auto& v = NUdf::TUnboxedValuePod(ui64("123456789ULL")); + TType* type = PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id); + TestPackPerformance(type, v); + } + + void TestRopeSplit() { + if constexpr (Transport) { + TType* tupleType; + const auto value = MakeTupleValue(tupleType); + + TValuePackerType packer(false, tupleType); + + auto buffer = packer.Pack(value); + + TString packed = buffer.ConvertToString(); + + if constexpr (Fast) { + UNIT_ASSERT_VALUES_EQUAL(packed.size(), 59); + } else { + UNIT_ASSERT_VALUES_EQUAL(packed.size(), 43); + } + + for (size_t chunk = 1; chunk < packed.size(); ++chunk) { + TString first = packed.substr(0, chunk); + TString second = packed.substr(chunk); + + TRope result(std::move(first)); + result.Insert(result.End(), TRope(std::move(second))); + + UNIT_ASSERT_VALUES_EQUAL(result.size(), packed.size()); + UNIT_ASSERT(!result.IsContiguous()); + + ValidateTupleValue(packer.Unpack(std::move(result), HolderFactory)); + } + } + } + + void TestIncrementalPacking() { + if constexpr (Transport) { + auto itemType = PgmBuilder.NewDataType(NUdf::TDataType<char *>::Id); + auto listType = PgmBuilder.NewListType(itemType); + TValuePackerType packer(false, itemType); + TValuePackerType listPacker(false, listType); + + TStringBuf str = "01234567890ABCDEFG"; + + size_t count = 500000; + + for (size_t i = 0; i < count; ++i) { + NUdf::TUnboxedValue item(MakeString(str)); + packer.AddItem(item); + } + + auto serialized = packer.Finish(); + + auto listObj = listPacker.Unpack(TRope(serialized), HolderFactory); + UNIT_ASSERT_VALUES_EQUAL(listObj.GetListLength(), count); + ui32 i = 0; + const auto iter = listObj.GetListIterator(); + for (NUdf::TUnboxedValue uVal; iter.Next(uVal); ++i) { + UNIT_ASSERT(uVal); + UNIT_ASSERT_VALUES_EQUAL(std::string_view(uVal.AsStringRef()), str); + } + + TUnboxedValueBatch items; + packer.UnpackBatch(std::move(serialized), HolderFactory, items); + UNIT_ASSERT_VALUES_EQUAL(items.RowCount(), count); + items.ForEachRow([&](const NUdf::TUnboxedValue& value) { + UNIT_ASSERT(value); + UNIT_ASSERT_VALUES_EQUAL(std::string_view(value.AsStringRef()), str); + }); + } + } + + void DoTestBlockPacking(ui64 offset, ui64 len) { + if constexpr (Transport) { + auto strType = PgmBuilder.NewDataType(NUdf::TDataType<char*>::Id); + auto ui32Type = PgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id); + auto ui64Type = PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id); + auto optStrType = PgmBuilder.NewOptionalType(strType); + auto optUi32Type = PgmBuilder.NewOptionalType(ui32Type); + + auto tupleOptUi32StrType = PgmBuilder.NewTupleType({ optUi32Type, strType }); + auto optTupleOptUi32StrType = PgmBuilder.NewOptionalType(tupleOptUi32StrType); + + auto blockUi32Type = PgmBuilder.NewBlockType(ui32Type, TBlockType::EShape::Many); + auto blockOptStrType = PgmBuilder.NewBlockType(optStrType, TBlockType::EShape::Many); + auto scalarOptStrType = PgmBuilder.NewBlockType(optStrType, TBlockType::EShape::Scalar); + auto blockOptTupleOptUi32StrType = PgmBuilder.NewBlockType(optTupleOptUi32StrType, TBlockType::EShape::Many); + auto scalarUi64Type = PgmBuilder.NewBlockType(ui64Type, TBlockType::EShape::Scalar); + + auto rowType = PgmBuilder.NewMultiType({ blockUi32Type, blockOptStrType, scalarOptStrType, blockOptTupleOptUi32StrType, scalarUi64Type }); + + + ui64 blockLen = 1000; + UNIT_ASSERT_LE(offset + len, blockLen); + + auto builder1 = MakeArrayBuilder(TTypeInfoHelper(), ui32Type, *ArrowPool_, CalcBlockLen(CalcMaxBlockItemSize(ui32Type)), nullptr); + auto builder2 = MakeArrayBuilder(TTypeInfoHelper(), optStrType, *ArrowPool_, CalcBlockLen(CalcMaxBlockItemSize(optStrType)), nullptr); + auto builder3 = MakeArrayBuilder(TTypeInfoHelper(), optTupleOptUi32StrType, *ArrowPool_, CalcBlockLen(CalcMaxBlockItemSize(optTupleOptUi32StrType)), nullptr); + + for (ui32 i = 0; i < blockLen; ++i) { + TBlockItem b1(i); + builder1->Add(b1); + + TString a = "a string " + ToString(i); + TBlockItem b2 = (i % 2) ? TBlockItem(a) : TBlockItem(); + builder2->Add(b2); + + TBlockItem b3items[] = { (i % 2) ? TBlockItem(i) : TBlockItem(), TBlockItem(a) }; + TBlockItem b3 = (i % 7) ? TBlockItem(b3items) : TBlockItem(); + builder3->Add(b3); + } + + std::string_view testScalarString = "foobar"; + auto strbuf = std::make_shared<arrow::Buffer>((const ui8*)testScalarString.data(), testScalarString.size()); + + TVector<arrow::Datum> datums; + datums.emplace_back(builder1->Build(true)); + datums.emplace_back(builder2->Build(true)); + datums.emplace_back(arrow::Datum(std::make_shared<arrow::BinaryScalar>(strbuf))); + datums.emplace_back(builder3->Build(true)); + datums.emplace_back(arrow::Datum(std::make_shared<arrow::UInt64Scalar>(blockLen))); + + if (offset != 0 || len != blockLen) { + for (auto& datum : datums) { + if (datum.is_array()) { + datum = NYql::NUdf::DeepSlice(datum.array(), offset, len); + } + } + } + TUnboxedValueVector columns; + for (auto& datum : datums) { + columns.emplace_back(HolderFactory.CreateArrowBlock(std::move(datum))); + } + + TValuePackerType packer(false, rowType, ArrowPool_); + packer.AddWideItem(columns.data(), columns.size()); + TRope packed = packer.Finish(); + + TUnboxedValueBatch unpacked(rowType); + packer.UnpackBatch(std::move(packed), HolderFactory, unpacked); + + UNIT_ASSERT_VALUES_EQUAL(unpacked.RowCount(), 1); + + TUnboxedValueVector unpackedColumns; + unpacked.ForEachRowWide([&](const NYql::NUdf::TUnboxedValue* values, ui32 count) { + unpackedColumns.insert(unpackedColumns.end(), values, values + count); + }); + + UNIT_ASSERT_VALUES_EQUAL(unpackedColumns.size(), columns.size()); + UNIT_ASSERT_VALUES_EQUAL(TArrowBlock::From(unpackedColumns.back()).GetDatum().scalar_as<arrow::UInt64Scalar>().value, blockLen); + UNIT_ASSERT_VALUES_EQUAL(TArrowBlock::From(unpackedColumns[2]).GetDatum().scalar_as<arrow::BinaryScalar>().value->ToString(), testScalarString); + + + auto reader1 = MakeBlockReader(TTypeInfoHelper(), ui32Type); + auto reader2 = MakeBlockReader(TTypeInfoHelper(), optStrType); + auto reader3 = MakeBlockReader(TTypeInfoHelper(), optTupleOptUi32StrType); + + for (ui32 i = offset; i < len; ++i) { + TBlockItem b1 = reader1->GetItem(*TArrowBlock::From(unpackedColumns[0]).GetDatum().array(), i - offset); + UNIT_ASSERT_VALUES_EQUAL(b1.As<ui32>(), i); + + TString a = "a string " + ToString(i); + TBlockItem b2 = reader2->GetItem(*TArrowBlock::From(unpackedColumns[1]).GetDatum().array(), i - offset); + if (i % 2) { + UNIT_ASSERT_VALUES_EQUAL(std::string_view(b2.AsStringRef()), a); + } else { + UNIT_ASSERT(!b2); + } + + TBlockItem b3 = reader3->GetItem(*TArrowBlock::From(unpackedColumns[3]).GetDatum().array(), i - offset); + if (i % 7) { + auto elements = b3.GetElements(); + if (i % 2) { + UNIT_ASSERT_VALUES_EQUAL(elements[0].As<ui32>(), i); + } else { + UNIT_ASSERT(!elements[0]); + } + UNIT_ASSERT_VALUES_EQUAL(std::string_view(elements[1].AsStringRef()), a); + } else { + UNIT_ASSERT(!b3); + } + } + } + } + + void TestBlockPacking() { + DoTestBlockPacking(0, 1000); + } + + void TestBlockPackingSliced() { + DoTestBlockPacking(19, 623); + } +private: + TIntrusivePtr<NMiniKQL::IFunctionRegistry> FunctionRegistry; + TIntrusivePtr<IRandomProvider> RandomProvider; + TScopedAlloc Alloc; + TTypeEnvironment Env; + TProgramBuilder PgmBuilder; + TMemoryUsageInfo MemInfo; + THolderFactory HolderFactory; + arrow::MemoryPool* const ArrowPool_; +}; + +class TMiniKQLComputationNodeGenericPackTest: public TMiniKQLComputationNodePackTest<false, false> { + UNIT_TEST_SUITE(TMiniKQLComputationNodeGenericPackTest); + UNIT_TEST(TestNumericTypes); + UNIT_TEST(TestOptionalNumericTypes); + UNIT_TEST(TestStringTypes); + UNIT_TEST(TestUuidType); + UNIT_TEST(TestOptionalStringTypes); + UNIT_TEST(TestListType); + UNIT_TEST(TestListOfOptionalsType); + UNIT_TEST(TestTupleType); + UNIT_TEST(TestStructType); + UNIT_TEST(TestOptionalType); + UNIT_TEST(TestDictType); + UNIT_TEST(TestVariantTypeOverStruct); + UNIT_TEST(TestVariantTypeOverTuple); + UNIT_TEST(TestIntegerPackPerformance); + UNIT_TEST(TestShortStringPackPerformance); + UNIT_TEST(TestPairPackPerformance); + UNIT_TEST(TestTuplePackPerformance); + UNIT_TEST_SUITE_END(); +}; + +class TMiniKQLComputationNodeGenericFastPackTest: public TMiniKQLComputationNodePackTest<true, false> { + UNIT_TEST_SUITE(TMiniKQLComputationNodeGenericFastPackTest); + UNIT_TEST(TestNumericTypes); + UNIT_TEST(TestOptionalNumericTypes); + UNIT_TEST(TestStringTypes); + UNIT_TEST(TestUuidType); + UNIT_TEST(TestOptionalStringTypes); + UNIT_TEST(TestListType); + UNIT_TEST(TestListOfOptionalsType); + UNIT_TEST(TestTupleType); + UNIT_TEST(TestStructType); + UNIT_TEST(TestOptionalType); + UNIT_TEST(TestDictType); + UNIT_TEST(TestVariantTypeOverStruct); + UNIT_TEST(TestVariantTypeOverTuple); + UNIT_TEST(TestIntegerPackPerformance); + UNIT_TEST(TestShortStringPackPerformance); + UNIT_TEST(TestPairPackPerformance); + UNIT_TEST(TestTuplePackPerformance); + UNIT_TEST_SUITE_END(); +}; + +class TMiniKQLComputationNodeTransportPackTest: public TMiniKQLComputationNodePackTest<false, true> { + UNIT_TEST_SUITE(TMiniKQLComputationNodeTransportPackTest); + UNIT_TEST(TestNumericTypes); + UNIT_TEST(TestOptionalNumericTypes); + UNIT_TEST(TestStringTypes); + UNIT_TEST(TestUuidType); + UNIT_TEST(TestOptionalStringTypes); + UNIT_TEST(TestListType); + UNIT_TEST(TestListOfOptionalsType); + UNIT_TEST(TestTupleType); + UNIT_TEST(TestStructType); + UNIT_TEST(TestOptionalType); + UNIT_TEST(TestDictType); + UNIT_TEST(TestVariantTypeOverStruct); + UNIT_TEST(TestVariantTypeOverTuple); + UNIT_TEST(TestIntegerPackPerformance); + UNIT_TEST(TestShortStringPackPerformance); + UNIT_TEST(TestPairPackPerformance); + UNIT_TEST(TestTuplePackPerformance); + UNIT_TEST(TestRopeSplit); + UNIT_TEST(TestIncrementalPacking); + UNIT_TEST(TestBlockPacking); + UNIT_TEST(TestBlockPackingSliced); + UNIT_TEST_SUITE_END(); +}; + +class TMiniKQLComputationNodeTransportFastPackTest: public TMiniKQLComputationNodePackTest<true, true> { + UNIT_TEST_SUITE(TMiniKQLComputationNodeTransportFastPackTest); + UNIT_TEST(TestNumericTypes); + UNIT_TEST(TestOptionalNumericTypes); + UNIT_TEST(TestStringTypes); + UNIT_TEST(TestUuidType); + UNIT_TEST(TestOptionalStringTypes); + UNIT_TEST(TestListType); + UNIT_TEST(TestListOfOptionalsType); + UNIT_TEST(TestTupleType); + UNIT_TEST(TestStructType); + UNIT_TEST(TestOptionalType); + UNIT_TEST(TestDictType); + UNIT_TEST(TestVariantTypeOverStruct); + UNIT_TEST(TestVariantTypeOverTuple); + UNIT_TEST(TestIntegerPackPerformance); + UNIT_TEST(TestShortStringPackPerformance); + UNIT_TEST(TestPairPackPerformance); + UNIT_TEST(TestTuplePackPerformance); + UNIT_TEST(TestRopeSplit); + UNIT_TEST(TestIncrementalPacking); + UNIT_TEST(TestBlockPacking); + UNIT_TEST(TestBlockPackingSliced); + UNIT_TEST_SUITE_END(); +}; + +UNIT_TEST_SUITE_REGISTRATION(TMiniKQLComputationNodeGenericPackTest); +UNIT_TEST_SUITE_REGISTRATION(TMiniKQLComputationNodeGenericFastPackTest); +UNIT_TEST_SUITE_REGISTRATION(TMiniKQLComputationNodeTransportPackTest); +UNIT_TEST_SUITE_REGISTRATION(TMiniKQLComputationNodeTransportFastPackTest); +} +} diff --git a/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache_ut.cpp b/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache_ut.cpp new file mode 100644 index 00000000000..f16ee88a84e --- /dev/null +++ b/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache_ut.cpp @@ -0,0 +1,837 @@ +#include "library/cpp/threading/local_executor/local_executor.h" +#include "ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.h" +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_pattern_cache.h> +#include <ydb/library/yql/minikql/mkql_type_builder.h> +#include <ydb/library/yql/minikql/mkql_node_serialization.h> +#include <ydb/library/yql/utils/yql_panic.h> +#include <ydb/library/yql/minikql/mkql_node.h> +#include <ydb/library/yql/minikql/mkql_program_builder.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node_impl.h> +#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> +#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h> +#include <ydb/library/yql/dq/proto/dq_tasks.pb.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/datetime/cputimer.h> + +namespace NKikimr { +namespace NMiniKQL { + +using namespace NYql::NUdf; + +TComputationNodeFactory GetListTestFactory() { + return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* { + if (callable.GetType()->GetName() == "TestList") { + return new TExternalComputationNode(ctx.Mutables); + } + return GetBuiltinFactory()(callable, ctx); + }; +} + +TRuntimeNode CreateFlow(TProgramBuilder& pb, size_t vecSize, TCallable* list) { + if (list) { + return pb.ToFlow(TRuntimeNode(list, false)); + } else { + std::vector<const TRuntimeNode> arr; + arr.reserve(vecSize); + for (ui64 i = 0; i < vecSize; ++i) { + arr.push_back(pb.NewDataLiteral<ui64>((i + 124515) % 6740234)); + } + TArrayRef<const TRuntimeNode> arrRef(std::move(arr)); + return pb.ToFlow(pb.AsList(arrRef)); + } +} + +template<bool Wide> +TRuntimeNode CreateFilter(TProgramBuilder& pb, size_t vecSize, TCallable* list); + +template<> +TRuntimeNode CreateFilter<false>(TProgramBuilder& pb, size_t vecSize, TCallable* list) { + TTimer t(TString(__func__) + ": "); + auto flow = CreateFlow(pb, vecSize, list); + + auto handler = [&](TRuntimeNode node) -> TRuntimeNode { + return pb.AggrEquals( + pb.Mod(node, pb.NewOptional(pb.NewDataLiteral<ui64>(128))), + pb.NewOptional(pb.NewDataLiteral<ui64>(0))); + }; + return pb.Filter(flow, handler); +} + +template<> +TRuntimeNode CreateFilter<true>(TProgramBuilder& pb, size_t vecSize, TCallable* list) { + TTimer t(TString(__func__) + ": "); + auto flow = CreateFlow(pb, vecSize, list); + + auto handler = [&](TRuntimeNode::TList node) -> TRuntimeNode { + return pb.AggrEquals( + pb.Mod(node.front(), pb.NewOptional(pb.NewDataLiteral<ui64>(128))), + pb.NewOptional(pb.NewDataLiteral<ui64>(0))); + }; + return pb.NarrowMap( + pb.WideFilter( + pb.ExpandMap(flow, + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; } + ), + handler + ), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.front(); } + ); +} + +template<bool Wide> +TRuntimeNode CreateMap(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr); + +template<> +TRuntimeNode CreateMap<false>(TProgramBuilder& pb, size_t vecSize, TCallable* list) { + TTimer t(TString(__func__) + ": "); + auto flow = CreateFlow(pb, vecSize, list); + + auto handler = [&](TRuntimeNode node) -> TRuntimeNode { + return pb.AggrEquals( + pb.Mod(node, pb.NewOptional(pb.NewDataLiteral<ui64>(128))), + pb.NewOptional(pb.NewDataLiteral<ui64>(0))); + }; + return pb.Map(flow, handler); +} + +template<> +TRuntimeNode CreateMap<true>(TProgramBuilder& pb, size_t vecSize, TCallable* list) { + TTimer t(TString(__func__) + ": "); + auto flow = CreateFlow(pb, vecSize, list); + + auto handler = [&](TRuntimeNode::TList node) -> TRuntimeNode::TList { + return {pb.AggrEquals( + pb.Mod(node.front(), pb.NewOptional(pb.NewDataLiteral<ui64>(128))), + pb.NewOptional(pb.NewDataLiteral<ui64>(0)))}; + }; + return pb.NarrowMap( + pb.WideMap( + pb.ExpandMap(flow, + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; } + ), + handler + ), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.front(); } + ); +} + +template<bool Wide> +TRuntimeNode CreateCondense(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr); + +template<> +TRuntimeNode CreateCondense<false>(TProgramBuilder& pb, size_t vecSize, TCallable* list) { + TTimer t(TString(__func__) + ": "); + auto flow = CreateFlow(pb, vecSize, list); + + auto switcherHandler = [&](TRuntimeNode, TRuntimeNode) -> TRuntimeNode { + return pb.NewDataLiteral<bool>(false); + }; + auto updateHandler = [&](TRuntimeNode item, TRuntimeNode state) -> TRuntimeNode { + return pb.Add(item, state); + }; + TRuntimeNode state = pb.NewDataLiteral<ui64>(0); + return pb.Condense(flow, state, switcherHandler, updateHandler); +} + +template<> +TRuntimeNode CreateCondense<true>(TProgramBuilder& pb, size_t vecSize, TCallable* list) { + TTimer t(TString(__func__) + ": "); + auto flow = CreateFlow(pb, vecSize, list); + + TRuntimeNode state = pb.NewDataLiteral<ui64>(0); + return pb.NarrowMap( + pb.WideCondense1( + /* stream */ + pb.ExpandMap(flow, + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; } + ), + /* init */ + [&](TRuntimeNode::TList item) -> TRuntimeNode::TList { return {item}; }, + /* switcher */ + [&](TRuntimeNode::TList, TRuntimeNode::TList) -> TRuntimeNode { return pb.NewDataLiteral<bool>(false); }, + /* handler */ + [&](TRuntimeNode::TList item, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.Add(item.front(), state.front())}; } + ), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.front(); } + ); +} + +template<bool Wide> +TRuntimeNode CreateChopper(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr); + +template<> +TRuntimeNode CreateChopper<false>(TProgramBuilder& pb, size_t vecSize, TCallable* list) { + TTimer t(TString(__func__) + ": "); + auto flow = CreateFlow(pb, vecSize, list); + + return pb.Chopper(flow, + /* keyExtractor */ + [&](TRuntimeNode item) -> TRuntimeNode { return item; }, + /* groupSwitch */ + [&](TRuntimeNode key, TRuntimeNode /*item*/) -> TRuntimeNode { + return pb.AggrEquals( + pb.Mod(key, pb.NewOptional(pb.NewDataLiteral<ui64>(128))), + pb.NewOptional(pb.NewDataLiteral<ui64>(0))); + }, + /* groupHandler */ + [&](TRuntimeNode, TRuntimeNode list) -> TRuntimeNode { return list; } + ); +}; + +template<> +TRuntimeNode CreateChopper<true>(TProgramBuilder& pb, size_t vecSize, TCallable* list) { + TTimer t(TString(__func__) + ": "); + auto flow = CreateFlow(pb, vecSize, list); + + return pb.NarrowMap( + pb.WideChopper( + /* stream */ + pb.ExpandMap(flow, + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; } + ), + /* keyExtractor */ + [&](TRuntimeNode::TList item) -> TRuntimeNode::TList { return item; }, + /* groupSwitch */ + [&](TRuntimeNode::TList key, TRuntimeNode::TList /*item*/) -> TRuntimeNode { + return pb.AggrEquals( + pb.Mod(key.front(), pb.NewOptional(pb.NewDataLiteral<ui64>(128))), + pb.NewOptional(pb.NewDataLiteral<ui64>(0))); + }, + /* groupHandler */ + [&](TRuntimeNode::TList, TRuntimeNode input) { return pb.WideMap(input, [](TRuntimeNode::TList items) { return items; }); } + ), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.front(); } + ); +}; + +template<bool Wide> +TRuntimeNode CreateCombine(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr); + +template<> +TRuntimeNode CreateCombine<false>(TProgramBuilder& pb, size_t vecSize, TCallable* list) { + TTimer t(TString(__func__) + ": "); + auto flow = CreateFlow(pb, vecSize, list); + + return pb.CombineCore( + /* stream */ + flow, + /* keyExtractor */ + [&] (TRuntimeNode /*item*/) -> TRuntimeNode { return pb.NewDataLiteral<ui64>(0);}, + /* init */ + [&] (TRuntimeNode /* key */, TRuntimeNode item) -> TRuntimeNode { return item; }, + /* update */ + [&] (TRuntimeNode /* key */, TRuntimeNode item, TRuntimeNode state) -> TRuntimeNode { return pb.Add(item, state); }, + /* finish */ + [&] (TRuntimeNode /* key */, TRuntimeNode item) -> TRuntimeNode { return pb.NewOptional(item); }, + /* memlimit */ + 64 << 20 + ); +}; + +template<> +TRuntimeNode CreateCombine<true>(TProgramBuilder& pb, size_t vecSize, TCallable* list) { + + TTimer t(TString(__func__) + ": "); + auto flow = CreateFlow(pb, vecSize, list); + + return pb.NarrowMap( + pb.WideCombiner( + /* stream */ + pb.ExpandMap(flow, + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; } + ), + /* memlimit */ + 64 << 20, + /* keyExtractor */ + [&] (TRuntimeNode::TList /*item*/) -> TRuntimeNode::TList { return {pb.NewDataLiteral<ui64>(0)};}, + /* init */ + [&] (TRuntimeNode::TList /* key */, TRuntimeNode::TList item) -> TRuntimeNode::TList { return {item}; }, + /* update */ + [&] (TRuntimeNode::TList /* key */, TRuntimeNode::TList item, TRuntimeNode::TList state) -> TRuntimeNode::TList { + return {pb.Add(item.front(), state.front())}; + }, + /* finish */ + [&] (TRuntimeNode::TList /* key */, TRuntimeNode::TList item) -> TRuntimeNode::TList { return {pb.NewOptional(item.front())}; } + ), + [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.front(); } + ); +}; + +template<bool Wide> +TRuntimeNode CreateChain1Map(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr); + +template<> +TRuntimeNode CreateChain1Map<false>(TProgramBuilder& pb, size_t vecSize, TCallable* list) { + TTimer t(TString(__func__) + ": "); + auto flow = CreateFlow(pb, vecSize, list); + + return pb.Chain1Map( + flow, + /* init */ + [&] (TRuntimeNode item) -> TRuntimeNode { return item; }, + /* update */ + [&] (TRuntimeNode item, TRuntimeNode state) -> TRuntimeNode { return pb.Add(item, state); } + ); +} + +template<> +TRuntimeNode CreateChain1Map<true>(TProgramBuilder& pb, size_t vecSize, TCallable* list) { + TTimer t(TString(__func__) + ": "); + auto flow = CreateFlow(pb, vecSize, list); + + return pb.NarrowMap( + pb.WideChain1Map( + /* stream */ + pb.ExpandMap(flow, + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; } + ), + /* init */ + [&] (TRuntimeNode::TList item) -> TRuntimeNode::TList { return item; }, + /* update */ + [&] (TRuntimeNode::TList item, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.Add(item.front(), state.front())}; } + ), + [&] (TRuntimeNode::TList item) -> TRuntimeNode { return item.front(); } + ); +} + +template<bool Wide> +TRuntimeNode CreateDiscard(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr) { + TTimer t(TString(__func__) + ": "); + auto flow = CreateFlow(pb, vecSize, list); + + if (Wide) { + return pb.Discard( + pb.ExpandMap(flow, + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; } + ) + ); + } else { + return pb.Discard(flow); + } +} + +template<bool Wide> +TRuntimeNode CreateSkip(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr) { + TTimer t(TString(__func__) + ": "); + auto flow = CreateFlow(pb, vecSize, list); + + auto count = pb.NewDataLiteral<ui64>(500); + if (Wide) { + return pb.NarrowMap( + pb.Skip( + pb.ExpandMap(flow, + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; } + ), + count + ), + [&] (TRuntimeNode::TList item) -> TRuntimeNode { return item.front(); } + ); + } else { + return pb.Skip(flow, count); + } +} + +template<bool Flow> +TRuntimeNode CreateNarrowFlatMap(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr) { + TTimer t(TString(__func__) + ": "); + auto flow = CreateFlow(pb, vecSize, list); + + return pb.NarrowFlatMap( + pb.ExpandMap(flow, + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; } + ), + [&] (TRuntimeNode::TList item) -> TRuntimeNode { + auto x = pb.NewOptional(item.front()); + return Flow ? pb.ToFlow(x) : x; + } + ); +} + +TRuntimeNode CreateNarrowMultiMap(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr) { + TTimer t(TString(__func__) + ": "); + auto flow = CreateFlow(pb, vecSize, list); + + return pb.NarrowMultiMap( + pb.ExpandMap(flow, + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; } + ), + [&] (TRuntimeNode::TList item) -> TRuntimeNode::TList { + return {item.front(), item.front()}; + } + ); +} + +template<bool WithPayload> +TRuntimeNode CreateSqueezeToSortedDict(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr) { + TTimer t(TString(__func__) + ": "); + auto flow = CreateFlow(pb, vecSize, list); + + return pb.FlatMap( + pb.NarrowSqueezeToSortedDict( + pb.ExpandMap(flow, + [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; } + ), + /*all*/ false, + /*keySelector*/ [&](TRuntimeNode::TList item) { return item.front(); }, + /*payloadSelector*/ [&](TRuntimeNode::TList ) { return WithPayload ? pb.NewDataLiteral<ui64>(0) : pb.NewVoid(); } + ), + [&] (TRuntimeNode item) { return pb.DictKeys(item); } + ); +} + +TRuntimeNode CreateMapJoin(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr) { + TTimer t(TString(__func__) + ": "); + auto flow = CreateFlow(pb, vecSize, list); + + const auto tupleType = pb.NewTupleType({ + pb.NewDataType(NUdf::TDataType<ui32>::Id), + pb.NewDataType(NUdf::TDataType<ui64>::Id) + }); + + const auto list1 = pb.Map(flow, [&] (TRuntimeNode item) { + return pb.NewTuple({pb.Mod(item, pb.NewDataLiteral<ui64>(1000)), pb.NewDataLiteral<ui32>(1)}); + }); + + const auto list2 = pb.NewList(tupleType, { + pb.NewTuple({pb.NewDataLiteral<ui32>(1), pb.NewDataLiteral<ui64>(3 * 1000)}), + pb.NewTuple({pb.NewDataLiteral<ui32>(2), pb.NewDataLiteral<ui64>(4 * 1000)}), + pb.NewTuple({pb.NewDataLiteral<ui32>(3), pb.NewDataLiteral<ui64>(5 * 1000)}), + }); + + const auto dict = pb.ToSortedDict(list2, false, + [&](TRuntimeNode item) { + return pb.Nth(item, 0); + }, + [&](TRuntimeNode item) { + return pb.NewTuple({pb.Nth(item, 1U)}); + }); + + const auto resultType = pb.NewFlowType(pb.NewMultiType({ + pb.NewDataType(NUdf::TDataType<char*>::Id), + pb.NewDataType(NUdf::TDataType<char*>::Id), + })); + + return pb.Map( + pb.NarrowMap(pb.MapJoinCore( + pb.ExpandMap(list1, [&] (TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0), pb.Nth(item, 1)}; }), + dict, + EJoinKind::Inner, + {0U}, + {1U, 0U}, + {0U, 1U}, + resultType + ), + [&](TRuntimeNode::TList items) { return pb.NewTuple(items); } + ), + [&](TRuntimeNode item) { return pb.Nth(item, 1); } + ); +} + +Y_UNIT_TEST_SUITE(ComputationGraphDataRace) { + template<class T> + void ParallelProgTest(T f, bool useLLVM, ui64 testResult, size_t vecSize = 10'000) { + TTimer t("total: "); + const ui32 cacheSizeInBytes = 104857600; // 100 MiB + const ui32 inFlight = 7; + TComputationPatternLRUCache cache(cacheSizeInBytes); + + auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry())->Clone(); + auto entry = std::make_shared<TPatternCacheEntry>(); + TScopedAlloc& alloc = entry->Alloc; + TTypeEnvironment& typeEnv = entry->Env; + + TProgramBuilder pb(typeEnv, *functionRegistry); + + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<ui64>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + TRuntimeNode progReturn; + with_lock(alloc) { + progReturn = f(pb, vecSize, list); + } + + TExploringNodeVisitor explorer; + explorer.Walk(progReturn.GetNode(), typeEnv); + + TComputationPatternOpts opts(alloc.Ref(), typeEnv, GetListTestFactory(), functionRegistry.Get(), + NUdf::EValidateMode::Lazy, NUdf::EValidatePolicy::Exception, useLLVM ? "" : "OFF", EGraphPerProcess::Multi); + + { + auto guard = entry->Env.BindAllocator(); + entry->Pattern = MakeComputationPattern(explorer, progReturn, {list}, opts); + } + cache.EmplacePattern("a", entry); + auto genData = [&]() { + std::vector<ui64> data; + data.reserve(vecSize); + for (ui64 i = 0; i < vecSize; ++i) { + data.push_back((i + 124515) % 6740234); + } + return data; + }; + + const auto data = genData(); + + std::vector<std::vector<ui64>> results(inFlight); + + NPar::LocalExecutor().RunAdditionalThreads(inFlight); + NPar::LocalExecutor().ExecRange([&](int id) { + for (ui32 i = 0; i < 100; ++i) { + auto key = "a"; + + auto randomProvider = CreateDeterministicRandomProvider(1); + auto timeProvider = CreateDeterministicTimeProvider(10000000); + TScopedAlloc graphAlloc(__LOCATION__); + + auto entry = cache.Find(key); + + TComputationPatternOpts opts(entry->Alloc.Ref(), entry->Env, GetListTestFactory(), + functionRegistry.Get(), NUdf::EValidateMode::Lazy, NUdf::EValidatePolicy::Exception, + useLLVM ? "" : "OFF", EGraphPerProcess::Multi); + + auto graph = entry->Pattern->Clone(opts.ToComputationOptions(*randomProvider, *timeProvider, &graphAlloc.Ref())); + TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(data.size(), items)); + + std::transform(data.cbegin(), data.cend(), items, + [](const auto s) { + return ToValue<ui64>(s); + }); + + ui64 acc = 0; + TUnboxedValue v = graph->GetValue(); + while (v.HasValue()) { + acc += v.Get<ui64>(); + v = graph->GetValue(); + } + results[id].push_back(acc); + } + }, 0, inFlight, NPar::TLocalExecutor::WAIT_COMPLETE | NPar::TLocalExecutor::MED_PRIORITY); + + for (auto threadResults : results) { + for (auto res : threadResults) { + UNIT_ASSERT_VALUES_EQUAL(res, testResult); + } + } + } + + Y_UNIT_TEST_QUAD(Filter, Wide, UseLLVM) { + ParallelProgTest(CreateFilter<Wide>, UseLLVM, 10098816); + } + + Y_UNIT_TEST_QUAD(Map, Wide, UseLLVM) { + ParallelProgTest(CreateMap<Wide>, UseLLVM, 78); + } + + Y_UNIT_TEST_QUAD(Condense, Wide, UseLLVM) { + ParallelProgTest(CreateCondense<Wide>, UseLLVM, 1295145000); + } + + Y_UNIT_TEST_QUAD(Chopper, Wide, UseLLVM) { + ParallelProgTest(CreateChopper<Wide>, UseLLVM, 1295145000); + } + + Y_UNIT_TEST_QUAD(Combine, Wide, UseLLVM) { + ParallelProgTest(CreateCombine<Wide>, UseLLVM, 1295145000); + } + + Y_UNIT_TEST_QUAD(Chain1Map, Wide, UseLLVM) { + ParallelProgTest(CreateChain1Map<Wide>, UseLLVM, 6393039240000); + } + + Y_UNIT_TEST_QUAD(Discard, Wide, UseLLVM) { + ParallelProgTest(CreateDiscard<Wide>, UseLLVM, 0); + } + + Y_UNIT_TEST_QUAD(Skip, Wide, UseLLVM) { + ParallelProgTest(CreateSkip<Wide>, UseLLVM, 1232762750); + } + + Y_UNIT_TEST_QUAD(NarrowFlatMap, Flow, UseLLVM) { + ParallelProgTest(CreateNarrowFlatMap<Flow>, UseLLVM, 1295145000); + } + + Y_UNIT_TEST_TWIN(NarrowMultiMap, UseLLVM) { + ParallelProgTest(CreateNarrowMultiMap, UseLLVM, 1295145000ull * 2); + } + + Y_UNIT_TEST_QUAD(SqueezeToSortedDict, WithPayload, UseLLVM) { + ParallelProgTest(CreateSqueezeToSortedDict<WithPayload>, UseLLVM, 125014500, 1000); + } + + Y_UNIT_TEST_TWIN(MapJoin, UseLLVM) { + ParallelProgTest(CreateMapJoin, UseLLVM, 120000, 10'000); + } +} + + +Y_UNIT_TEST_SUITE(ComputationPatternCache) { + Y_UNIT_TEST(Smoke) { + const ui32 cacheSize = 10'000'000; + const ui32 cacheItems = 10; + TComputationPatternLRUCache cache(cacheSize); + + auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry())->Clone(); + + for (ui32 i = 0; i < cacheItems; ++i) { + auto entry = std::make_shared<TPatternCacheEntry>(); + TScopedAlloc& alloc = entry->Alloc; + TTypeEnvironment& typeEnv = entry->Env; + + TProgramBuilder pb(typeEnv, *functionRegistry); + + TRuntimeNode progReturn; + with_lock(alloc) { + progReturn = pb.NewDataLiteral<NYql::NUdf::EDataSlot::String>("qwerty"); + } + + TExploringNodeVisitor explorer; + explorer.Walk(progReturn.GetNode(), typeEnv); + + TComputationPatternOpts opts(alloc.Ref(), typeEnv, GetBuiltinFactory(), + functionRegistry.Get(), NUdf::EValidateMode::Lazy, NUdf::EValidatePolicy::Exception, + "OFF", EGraphPerProcess::Multi); + + { + auto guard = entry->Env.BindAllocator(); + entry->Pattern = MakeComputationPattern(explorer, progReturn, {}, opts); + } + cache.EmplacePattern(TString((char)('a' + i)), entry); + } + + for (ui32 i = 0; i < cacheItems; ++i) { + auto key = TString((char)('a' + i)); + + auto randomProvider = CreateDeterministicRandomProvider(1); + auto timeProvider = CreateDeterministicTimeProvider(10000000); + TScopedAlloc graphAlloc(__LOCATION__); + auto entry = cache.Find(key); + UNIT_ASSERT(entry); + TComputationPatternOpts opts(entry->Alloc.Ref(), entry->Env, GetBuiltinFactory(), + functionRegistry.Get(), NUdf::EValidateMode::Lazy, NUdf::EValidatePolicy::Exception, + "OFF", EGraphPerProcess::Multi); + + auto graph = entry->Pattern->Clone(opts.ToComputationOptions(*randomProvider, *timeProvider, &graphAlloc.Ref())); + auto value = graph->GetValue(); + UNIT_ASSERT_EQUAL(value.AsStringRef(), NYql::NUdf::TStringRef("qwerty")); + } + } + + Y_UNIT_TEST(AddPerf) { + TTimer t("all: "); + TScopedAlloc alloc(__LOCATION__); + TTypeEnvironment typeEnv(alloc); + + auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry())->Clone(); + + TProgramBuilder pb(typeEnv, *functionRegistry); + auto prog1 = pb.NewDataLiteral<ui64>(123591592ULL); + auto prog2 = pb.NewDataLiteral<ui64>(323591592ULL); + auto progReturn = pb.Add(prog1, prog2); + + TExploringNodeVisitor explorer; + explorer.Walk(progReturn.GetNode(), typeEnv); + + NUdf::EValidateMode validateMode = NUdf::EValidateMode::Lazy; + TComputationPatternOpts opts(alloc.Ref(), typeEnv, GetBuiltinFactory(), + functionRegistry.Get(), validateMode, NUdf::EValidatePolicy::Exception, + "OFF", EGraphPerProcess::Multi); + + auto t_make_pattern = std::make_unique<TTimer>("make_pattern: "); + auto pattern = MakeComputationPattern(explorer, progReturn, {}, opts); + t_make_pattern.reset(); + auto randomProvider = CreateDeterministicRandomProvider(1); + auto timeProvider = CreateDeterministicTimeProvider(10000000); + + auto t_clone = std::make_unique<TTimer>("clone: "); + auto graph = pattern->Clone(opts.ToComputationOptions(*randomProvider, *timeProvider)); + t_clone.reset(); + + const ui64 repeats = 100'000; + + { + TTimer t("graph: "); + ui64 acc = 0; + for (ui64 i = 0; i < repeats; ++i) { + acc += graph->GetValue().Get<ui64>(); + } + Y_DO_NOT_OPTIMIZE_AWAY(acc); + } + { + std::function<ui64(ui64, ui64)> add = [](ui64 a, ui64 b) { + return a + b; + }; + + TTimer t("lambda: "); + ui64 acc = 0; + for (ui64 i = 0; i < repeats; ++i) { + acc += add(123591592ULL, 323591592ULL); + } + Y_DO_NOT_OPTIMIZE_AWAY(acc); + } + { + std::function<TUnboxedValue(TUnboxedValue&, TUnboxedValue&)> add = + [](TUnboxedValue& a, TUnboxedValue& b) { + return TUnboxedValuePod(a.Get<ui64>() + b.Get<ui64>()); + }; + Y_DO_NOT_OPTIMIZE_AWAY(add); + + TTimer t("lambda unboxed value: "); + TUnboxedValue acc(TUnboxedValuePod(0)); + TUnboxedValue v1(TUnboxedValuePod(ui64{123591592UL})); + TUnboxedValue v2(TUnboxedValuePod(ui64{323591592UL})); + for (ui64 i = 0; i < repeats; ++i) { + auto r = add(v1, v2); + acc = add(r, acc); + } + Y_DO_NOT_OPTIMIZE_AWAY(acc.Get<ui64>()); + } + } + + Y_UNIT_TEST_TWIN(FilterPerf, Wide) { + TScopedAlloc alloc(__LOCATION__); + TTypeEnvironment typeEnv(alloc); + + auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry())->Clone(); + + TProgramBuilder pb(typeEnv, *functionRegistry); + const ui64 vecSize = 100'000; + Cerr << "vecSize: " << vecSize << Endl; + const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<ui64>::Id)); + const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build(); + auto progReturn = CreateFilter<Wide>(pb, vecSize, list); + + TExploringNodeVisitor explorer; + explorer.Walk(progReturn.GetNode(), typeEnv); + + NUdf::EValidateMode validateMode = NUdf::EValidateMode::Max; + TComputationPatternOpts opts(alloc.Ref(), typeEnv, GetListTestFactory(), + functionRegistry.Get(), validateMode, NUdf::EValidatePolicy::Exception, + "OFF", EGraphPerProcess::Multi); + + auto t_make_pattern = std::make_unique<TTimer>("make_pattern: "); + auto pattern = MakeComputationPattern(explorer, progReturn, {list}, opts); + t_make_pattern.reset(); + + auto randomProvider = CreateDeterministicRandomProvider(1); + auto timeProvider = CreateDeterministicTimeProvider(10000000); + + auto t_clone = std::make_unique<TTimer>("clone: "); + auto graph = pattern->Clone(opts.ToComputationOptions(*randomProvider, *timeProvider)); + + t_clone.reset(); + + auto genData = [&]() { + std::vector<ui64> data; + data.reserve(vecSize); + for (ui64 i = 0; i < vecSize; ++i) { + data.push_back((i + 124515) % 6740234); + } + return data; + }; + + auto testResult = [&] (ui64 acc, ui64 count) { + if (vecSize == 100'000'000) { + UNIT_ASSERT_VALUES_EQUAL(acc, 2614128386688); + UNIT_ASSERT_VALUES_EQUAL(count, 781263); + } else if (vecSize == 10'000'000) { + UNIT_ASSERT_VALUES_EQUAL(acc, 222145217664); + } else if (vecSize == 100'000) { + UNIT_ASSERT_VALUES_EQUAL(acc, 136480896); + UNIT_ASSERT_VALUES_EQUAL(count, 782); + } else { + UNIT_FAIL("result is not checked"); + } + }; + + ui64 kIter = 2; + { + TDuration total; + for (ui64 i = 0; i < kIter; ++i) { + ui64 acc = 0; + ui64 count = 0; + + auto graph = pattern->Clone(opts.ToComputationOptions(*randomProvider, *timeProvider)); + auto data = genData(); + TUnboxedValue* items = nullptr; + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(data.size(), items)); + + std::transform(data.cbegin(), data.cend(), items, + [](const auto s) { + return ToValue<ui64>(s); + }); + + TSimpleTimer t; + TUnboxedValue v = graph->GetValue(); + while (v.HasValue()) { + acc += v.Get<ui64>(); + ++count; + v = graph->GetValue(); + } + testResult(acc, count); + + total += t.Get(); + } + Cerr << "graph: " << Sprintf("%.3f", total.SecondsFloat()) << "s" << Endl; + } + + { + auto data = genData(); + std::function<bool(ui64)> predicate = [](ui64 a) { + return a % 128 == 0; + }; + Y_DO_NOT_OPTIMIZE_AWAY(predicate); + + TDuration total; + + for (ui64 i = 0; i < kIter; ++i) { + TSimpleTimer t; + ui64 acc = 0; + ui64 count = 0; + for (ui64 j = 0; j < data.size(); ++j) { + if (predicate(data[j])) { + acc += data[j]; + ++count; + } + } + + total += t.Get(); + + testResult(acc, count); + } + Cerr << "std::function: " << Sprintf("%.3f", total.SecondsFloat()) << "s" << Endl; + } + + { + auto data = genData(); + auto predicate = [](ui64 a) { + return a % 128 == 0; + }; + Y_DO_NOT_OPTIMIZE_AWAY(predicate); + + TDuration total; + for (ui64 i = 0; i < kIter; ++i) { + TSimpleTimer t; + ui64 acc = 0; + ui64 count = 0; + for (ui64 j = 0; j < data.size(); ++j) { + if (predicate(data[j])) { + acc += data[j]; + ++count; + } + } + + total += t.Get(); + + testResult(acc, count); + } + Cerr << "lambda: " << Sprintf("%.3f", total.SecondsFloat()) << "s" << Endl; + } + } +} + +} +} diff --git a/ydb/library/yql/minikql/computation/mkql_validate_ut.cpp b/ydb/library/yql/minikql/computation/mkql_validate_ut.cpp new file mode 100644 index 00000000000..2dd6ec95e6c --- /dev/null +++ b/ydb/library/yql/minikql/computation/mkql_validate_ut.cpp @@ -0,0 +1,1153 @@ +#include <ydb/library/yql/minikql/mkql_program_builder.h> +#include <ydb/library/yql/minikql/mkql_node_printer.h> +#include "mkql_computation_list_adapter.h" +#include "mkql_computation_node_impl.h" +#include "mkql_computation_node.h" +#include "mkql_value_builder.h" +#include <ydb/library/yql/minikql/mkql_function_registry.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> +#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> +#include "mkql_validate.h" + +#include <ydb/library/yql/minikql/mkql_type_builder.h> +#include <ydb/library/yql/minikql/mkql_utils.h> +#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/generic/algorithm.h> + +#include <ydb/library/yql/public/udf/udf_helpers.h> + +namespace NYql { + +namespace { +using namespace NKikimr::NMiniKQL; +static const ui32 RAW_INDEX_NO_HOLE = -1; +static const ui32 RAW_BROKEN_INDEX_LIST_TO_DICT = 1; + +template<class T> +NUdf::TUnboxedValue ToUnboxedValue(const T& val) { + return NUdf::TUnboxedValuePod(val); +} + +NUdf::TUnboxedValue ToUnboxedValue(const TString& val) { + return MakeString(val); +} + +NUdf::TUnboxedValue ToUnboxedValue(const NUdf::IBoxedValuePtr& val) { + return NUdf::TUnboxedValuePod(NUdf::IBoxedValuePtr(val)); +} + +} // namespace NMiniKQL + +/// support for build Struct type @{ +namespace NUdf { + + template<class TContainer> + struct TListRefIterator: public TBoxedValue { + TListRefIterator(const TContainer& listRef, ui32 holePos) + : ListRef(listRef) + , Index(-1) + , HolePos(holePos) + {} + private: + const TContainer& ListRef; + ui32 Index; + ui32 HolePos; + + bool Next(NUdf::TUnboxedValue& value) final { + if (++Index >= ListRef.size()) + return false; + value = Index == HolePos ? NUdf::TUnboxedValue(NUdf::TUnboxedValuePod(42)) : ToUnboxedValue(ListRef[Index]); + return true; + } + }; + + template<class TContainer, ui32 TIndexDictBrokenHole = RAW_INDEX_NO_HOLE, bool TNoDictIndex = false> + struct TListRef: public NUdf::TBoxedValue { + TListRef(const TContainer& listRef, ui32 holePos = RAW_INDEX_NO_HOLE) + : ListRef(listRef) + , HolePos(holePos) + {} + + private: + const TContainer& ListRef; + const NUdf::IValueBuilder* ValueBuilder; + ui32 HolePos; + + bool HasFastListLength() const override { + return true; + } + + ui64 GetListLength() const override { + return ListRef.size(); + } + + ui64 GetEstimatedListLength() const override { + return ListRef.size(); + } + + NUdf::TUnboxedValue GetListIterator() const override { + return NUdf::TUnboxedValuePod(new TListRefIterator<TContainer>(ListRef, HolePos)); + } + + NUdf::IBoxedValuePtr ToIndexDictImpl(const IValueBuilder& builder) const override { + return TNoDictIndex ? nullptr : builder.ToIndexDict(NUdf::TUnboxedValuePod( + new TListRef<TContainer, TIndexDictBrokenHole, true>(ListRef, TIndexDictBrokenHole))).AsBoxed(); + } + }; + + struct PersonStruct { + static const size_t MEMBERS_COUNT = 3; + static ui32 MetaIndexes[MEMBERS_COUNT]; + static ui32 MetaBackIndexes[MEMBERS_COUNT]; + + TString FirstName; + TString LastName; + ui32 Age; + + NUdf::TUnboxedValue GetByIndex(ui32 index) const { + switch (index) { + case 0: return ToUnboxedValue(FirstName); + case 1: return ToUnboxedValue(LastName); + case 2: return NUdf::TUnboxedValuePod(Age); + default: Y_FAIL("Unexpected"); + } + } + }; + + ui32 PersonStruct::MetaIndexes[MEMBERS_COUNT]; + ui32 PersonStruct::MetaBackIndexes[MEMBERS_COUNT]; + + + struct PersonStructWithOptList { + static const size_t MEMBERS_COUNT = 4; + static ui32 MetaIndexes[MEMBERS_COUNT]; + static ui32 MetaBackIndexes[MEMBERS_COUNT]; + + TString FirstName; + TString LastName; + ui32 Age; + typedef std::vector<ui32> TTagList; + TTagList Tags; + + NUdf::TUnboxedValue GetByIndex(ui32 index) const { + switch (index) { + case 0: return ToUnboxedValue(FirstName); + case 1: return ToUnboxedValue(LastName); + case 2: return NUdf::TUnboxedValuePod(Age); + case 3: return Tags.empty() ? + NUdf::TUnboxedValuePod() : + NUdf::TUnboxedValuePod(new TListRef<TTagList>(Tags)); + default: Y_FAIL("Unexpected"); + } + } + }; + + ui32 PersonStructWithOptList::MetaIndexes[MEMBERS_COUNT]; + ui32 PersonStructWithOptList::MetaBackIndexes[MEMBERS_COUNT]; + + struct TCallableOneUi32Arg { + }; + +namespace NImpl { + + template<> + struct TTypeBuilderHelper<NUdf::PersonStruct> { + static TType* Build(const IFunctionTypeInfoBuilder& builder) { + auto structBuilder = builder.Struct(3); + structBuilder->AddField<char*>("FirstName", &PersonStruct::MetaIndexes[0]) + .AddField<char*>("LastName", &PersonStruct::MetaIndexes[1]) + .AddField<ui32>("Age", &PersonStruct::MetaIndexes[2]); + auto structType = structBuilder->Build(); + for (const auto& index: PersonStruct::MetaIndexes) { + Y_VERIFY(index < NUdf::PersonStruct::MEMBERS_COUNT); + NUdf::PersonStruct::MetaBackIndexes[index] = &index - PersonStruct::MetaIndexes; + Y_VERIFY(NUdf::PersonStruct::MetaBackIndexes[index] < NUdf::PersonStruct::MEMBERS_COUNT); + } + return structType; + } + }; + + template<> + struct TTypeBuilderHelper<NUdf::PersonStructWithOptList> { + static TType* Build(const IFunctionTypeInfoBuilder& builder) { + auto listTags = builder.List()->Item<ui32>().Build(); + auto optionalListTags = builder.Optional()->Item(listTags).Build(); + auto structBuilder = builder.Struct(3); + structBuilder->AddField<char*>("FirstName", &PersonStructWithOptList::MetaIndexes[0]) + .AddField<char*>("LastName", &PersonStructWithOptList::MetaIndexes[1]) + .AddField<ui32>("Age", &PersonStructWithOptList::MetaIndexes[2]) + .AddField("Tags", optionalListTags, &PersonStructWithOptList::MetaIndexes[3]); + auto structType = structBuilder->Build(); + for (const auto& index: PersonStructWithOptList::MetaIndexes) { + Y_VERIFY(index < NUdf::PersonStructWithOptList::MEMBERS_COUNT); + NUdf::PersonStructWithOptList::MetaBackIndexes[index] = &index - PersonStructWithOptList::MetaIndexes; + Y_VERIFY(NUdf::PersonStructWithOptList::MetaBackIndexes[index] < NUdf::PersonStructWithOptList::MEMBERS_COUNT); + } + return structType; + } + }; + + template<> + struct TTypeBuilderHelper<NUdf::TCallableOneUi32Arg> { + static TType* Build(const IFunctionTypeInfoBuilder& builder) { + auto callableBuilder = builder.Callable(1); + callableBuilder->Returns<ui32>(); + callableBuilder->Arg<ui32>(); + return callableBuilder->Build(); + } + }; +} // namespace NImpl +} // namespace NUdf +/// @} + + + struct TBrokenSeqListIterator: public NUdf::TBoxedValue { + TBrokenSeqListIterator(ui32 size, ui32 holePos) + : Size(size) + , HolePos(holePos) + , Index(-1) + {} + private: + ui32 Size; + ui32 HolePos; + ui32 Index; + + bool Skip() final { + return ++Index < Size; + } + + bool Next(NUdf::TUnboxedValue& value) final { + if (!Skip()) + return false; + value = Index == HolePos ? NUdf::TUnboxedValuePod() : NUdf::TUnboxedValuePod(Index); + return true; + } + }; + + struct TBrokenSeqListBoxedValue: public NUdf::TBoxedValue { + TBrokenSeqListBoxedValue(ui32 size, ui32 holePos) + : ListSize(size) + , HolePos(holePos) + {} + + private: + ui32 ListSize; + ui32 HolePos; + + bool HasFastListLength() const override { + return true; + } + + ui64 GetListLength() const override { + return ListSize; + } + + ui64 GetEstimatedListLength() const override { + return ListSize; + } + + NUdf::TUnboxedValue GetListIterator() const override { + return NUdf::TUnboxedValuePod(new TBrokenSeqListIterator(ListSize, HolePos)); + } + }; + + template<class TStructType> + struct TBrokenStructBoxedValue: public NUdf::TBoxedValue { + TBrokenStructBoxedValue(const TStructType& data, ui32 holePos = RAW_INDEX_NO_HOLE) + : Struct(data) + , HolePos(holePos) + {} + + private: + const TStructType& Struct; + ui32 HolePos; + + NUdf::TUnboxedValue GetElement(ui32 index) const override { + if (index == HolePos) { + return NUdf::TUnboxedValuePod(); + } + return Struct.GetByIndex(TStructType::MetaBackIndexes[index]); + } + }; + + +namespace { + template<> + NUdf::TUnboxedValue ToUnboxedValue<NUdf::PersonStruct>(const NUdf::PersonStruct& val) { + return NUdf::TUnboxedValuePod(new TBrokenStructBoxedValue<NUdf::PersonStruct>(val)); + } + + template<> + NUdf::TUnboxedValue ToUnboxedValue<NUdf::PersonStructWithOptList>(const NUdf::PersonStructWithOptList& val) { + return NUdf::TUnboxedValuePod(new TBrokenStructBoxedValue<NUdf::PersonStructWithOptList>(val)); + } + + template<class TTupleType> + struct TBrokenTupleBoxedValue: public NUdf::TBoxedValue { + TBrokenTupleBoxedValue(const TTupleType& tuple, ui32 holePos) + : Tuple(tuple) + , HolePos(holePos) + {} + + private: + const TTupleType& Tuple; + ui32 HolePos; + + NUdf::TUnboxedValue GetElement(ui32 index) const override { + if (index == HolePos) { + return NUdf::TUnboxedValuePod(); + } + switch (index) { + case 0: return ToUnboxedValue(std::get<0>(Tuple)); + case 1: return ToUnboxedValue(std::get<1>(Tuple)); + case 2: return ToUnboxedValue(std::get<2>(Tuple)); + case 3: return ToUnboxedValue(std::get<3>(Tuple)); + default: Y_FAIL("Unexpected"); + } + } + }; + + typedef std::pair<ui32, ui32> PosPair; + + template<class TKey, class TValue> + struct TBrokenDictIterator: public NUdf::TBoxedValue { + TBrokenDictIterator(const std::vector<std::pair<TKey, TValue>>& dictData, PosPair holePos) + : DictData(dictData) + , HolePos(holePos) + , Index(-1) + {} + + private: + const std::vector<std::pair<TKey, TValue>>& DictData; + PosPair HolePos; + ui32 Index; + + bool Skip() final { + return ++Index < DictData.size(); + } + + bool Next(NUdf::TUnboxedValue& key) final { + if (!Skip()) + return false; + key = Index == HolePos.first ? NUdf::TUnboxedValuePod() : NUdf::TUnboxedValuePod(DictData[Index].first); + return true; + } + + bool NextPair(NUdf::TUnboxedValue& key, NUdf::TUnboxedValue& payload) final { + if (!Next(key)) + return false; + payload = Index == HolePos.second ? NUdf::TUnboxedValue() : ToUnboxedValue(DictData[Index].second); + return true; + } + }; + + template<class TKey, class TValue> + struct TBrokenDictBoxedValue: public NUdf::TBoxedValue { + TBrokenDictBoxedValue(const std::vector<std::pair<TKey, TValue>>& dictData, + PosPair holePos, NUdf::TUnboxedValue&& hole = NUdf::TUnboxedValuePod()) + : DictData(dictData) + , HolePos(holePos) + , Hole(std::move(hole)) + {} + + private: + const std::vector<std::pair<TKey, TValue>> DictData; + PosPair HolePos; + NUdf::TUnboxedValue Hole; + + NUdf::TUnboxedValue GetKeysIterator() const override { + return NUdf::TUnboxedValuePod(new TBrokenDictIterator<TKey, TValue>(DictData, HolePos)); + } + + NUdf::TUnboxedValue GetDictIterator() const override { + return NUdf::TUnboxedValuePod(new TBrokenDictIterator<TKey, TValue>(DictData, HolePos)); + } + }; + + struct TThrowerValue: public NUdf::TBoxedValue { + static long Count; + TThrowerValue(NUdf::IBoxedValuePtr&& owner = NUdf::IBoxedValuePtr()) + : Owner(std::move(owner)) + { ++Count; } + ~TThrowerValue() { --Count; } + private: + const NUdf::IBoxedValuePtr Owner; + + bool Skip() override { + ythrow yexception() << "Throw"; + } + + NUdf::TUnboxedValue GetListIterator() const override { + return NUdf::TUnboxedValuePod(new TThrowerValue(const_cast<TThrowerValue*>(this))); + } + }; + + SIMPLE_UDF(TException, NUdf::TListType<ui32>()) { + Y_UNUSED(valueBuilder); + Y_UNUSED(args); + return NUdf::TUnboxedValuePod(new TThrowerValue); + } + + long TThrowerValue::Count = 0L; + + SIMPLE_UDF(TVoid, void()) { + Y_UNUSED(valueBuilder); + Y_UNUSED(args); + return NUdf::TUnboxedValuePod::Void(); + } + + SIMPLE_UDF_RUN(TNonEmpty, ui32(), NUdf::TOptional<void>) { + Y_UNUSED(valueBuilder); + Y_UNUSED(args); + return NUdf::TUnboxedValuePod(42); + } + + SIMPLE_UDF(TOptionalNonEmpty, NUdf::TOptional<ui32>()) { + Y_UNUSED(valueBuilder); + Y_UNUSED(args); + return NUdf::TUnboxedValuePod(42); + } + + SIMPLE_UDF_RUN(TOptionalEmpty, NUdf::TOptional<ui32>(), NUdf::TOptional<void>) { + Y_UNUSED(valueBuilder); + Y_UNUSED(args); + return NUdf::TUnboxedValuePod(); + } + + SIMPLE_UDF(TSub2Mul2BrokenOnLess2, ui32(ui32)) { + Y_UNUSED(valueBuilder); + const ui32 arg = args[0].Get<ui32>(); + if (arg >= 2) { + return NUdf::TUnboxedValuePod((arg - 2) * 2); + } + return NUdf::TUnboxedValuePod(); + } + + SIMPLE_UDF_RUN(TBackSub2Mul2, ui32(NUdf::TCallableOneUi32Arg, ui32), NUdf::TOptional<void>) { + const auto func = args[0]; + const auto& arg = args[1]; + auto usedArg = NUdf::TUnboxedValuePod(); + if (arg.Get<ui32>() < 100) { + usedArg = arg; + } + auto funcResult = func.Run(valueBuilder, &usedArg); + const auto& backResult = funcResult.Get<ui32>() / 2 + 2; + return NUdf::TUnboxedValuePod(backResult); + } + + SIMPLE_UDF(TSeqList, NUdf::TListType<ui32>(ui32)) { + const ui32 size = args[0].Get<ui32>(); + std::vector<NUdf::TUnboxedValue> res; + res.resize(size); + for (ui32 i = 0; i < size; ++i) { + res[i] = NUdf::TUnboxedValuePod(i); + } + return valueBuilder->NewList(res.data(), res.size()); + } + + SIMPLE_UDF_RUN(TSeqListWithHole, NUdf::TListType<ui32>(ui32, ui32), NUdf::TOptional<void>) { + Y_UNUSED(valueBuilder); + const ui32 size = args[0].Get<ui32>(); + const ui32 hole = args[1].Get<ui32>(); + NUdf::IBoxedValuePtr boxed(new TBrokenSeqListBoxedValue(size, hole)); + return NUdf::TUnboxedValuePod(std::move(boxed)); + } + + static const auto TUPLE = std::make_tuple(ui8(33), TString("world"), ui64(0xFEEDB00B2A115E), TString("funny bunny")); + + typedef NUdf::TTuple<ui8, char*, ui64, char*> NUdfTuple; + + SIMPLE_UDF(TTuple, NUdfTuple(ui32)) { + Y_UNUSED(valueBuilder); + const ui32 holePos = args[0].Get<ui32>(); + NUdf::IBoxedValuePtr boxed(new TBrokenTupleBoxedValue<decltype(TUPLE)>(TUPLE, holePos)); + return NUdf::TUnboxedValuePod(std::move(boxed)); + } + + static const std::vector<std::pair<ui32, ui64>> DICT_DIGIT2DIGIT = { + {1, 100500}, + {42, 0xDEADBEAF}, + {911, 1234567890}, + {777, 777777777777}, + }; + + typedef NUdf::TDict<ui32, ui64> NUdfDictDigDig; + + SIMPLE_UDF_RUN(TDictDigDig, NUdfDictDigDig(ui32, ui32), NUdf::TOptional<void>) { + Y_UNUSED(valueBuilder); + const ui32 holeKey = args[0].Get<ui32>(); + const ui32 holeValue = args[1].Get<ui32>(); + NUdf::IBoxedValuePtr boxed(new TBrokenDictBoxedValue<ui32, ui64>( + DICT_DIGIT2DIGIT, std::make_pair(holeKey, holeValue))); + return NUdf::TUnboxedValuePod(std::move(boxed)); + } + + SIMPLE_UDF(TDictDigDigHoleAsOpt, NUdfDictDigDig(ui32, ui32)) { + Y_UNUSED(valueBuilder); + const ui32 holeKey = args[0].Get<ui32>(); + const ui32 holeValue = args[1].Get<ui32>(); + NUdf::IBoxedValuePtr boxed(new TBrokenDictBoxedValue<ui32, ui64>(DICT_DIGIT2DIGIT, + std::make_pair(holeKey, holeValue), + NUdf::TUnboxedValuePod())); + return NUdf::TUnboxedValuePod(std::move(boxed)); + } + + static const NUdf::PersonStruct STRUCT_PERSON_JONNIE = {"Johnnie Walker", "Blue Label", 25}; + static const NUdf::PersonStruct STRUCT_PERSON_HITHCOCK = {"Alfred", "Hithcock", 81}; + static const NUdf::PersonStruct STRUCT_PERSON_LOVECRAFT = {"Howard", "Lovecraft", 25}; + static const NUdf::PersonStruct STRUCT_PERSON_KING = {"Stephen", "King", 25}; + static const NUdf::PersonStructWithOptList STRUCT_PERSON_HITHCOCK_LIST = {"Alfred", "Hithcock", 81, {}}; + static const NUdf::PersonStructWithOptList STRUCT_PERSON_LOVECRAFT_LIST = {"Howard", "Lovecraft", 25, {3, 2, 99}}; + static const NUdf::PersonStructWithOptList STRUCT_PERSON_KING_LIST = {"Stephen", "King", 25, {}}; + + SIMPLE_UDF_RUN(TPersonStruct, NUdf::PersonStruct(ui32), NUdf::TOptional<void>) { + Y_UNUSED(valueBuilder); + const ui32 holePos = args[0].Get<ui32>(); + NUdf::IBoxedValuePtr boxed(new TBrokenStructBoxedValue<NUdf::PersonStruct>(STRUCT_PERSON_JONNIE, holePos)); + return NUdf::TUnboxedValuePod(std::move(boxed)); + } + + typedef NUdf::TTuple<NUdf::PersonStructWithOptList,NUdf::PersonStruct,NUdf::PersonStructWithOptList,NUdf::PersonStruct> NUdfPersonTuple; + static const auto TUPLE_OF_PERSON = std::make_tuple( + STRUCT_PERSON_HITHCOCK_LIST, + STRUCT_PERSON_JONNIE, + STRUCT_PERSON_LOVECRAFT_LIST, + STRUCT_PERSON_KING); + + static const auto TUPLE_OF_PERSON_NO_LIST = std::make_tuple(STRUCT_PERSON_HITHCOCK_LIST, + STRUCT_PERSON_JONNIE, + STRUCT_PERSON_KING_LIST, + STRUCT_PERSON_KING); + + SIMPLE_UDF(TTupleOfPersonStruct, NUdfPersonTuple(ui32)) { + Y_UNUSED(valueBuilder); + const ui32 holePos = args[0].Get<ui32>(); + NUdf::IBoxedValuePtr boxed(new TBrokenTupleBoxedValue<decltype(TUPLE_OF_PERSON)>(TUPLE_OF_PERSON, holePos)); + return NUdf::TUnboxedValuePod(std::move(boxed)); + } + + SIMPLE_UDF(TTupleOfPersonStructNoList, NUdfPersonTuple(ui32)) { + Y_UNUSED(valueBuilder); + const ui32 holePos = args[0].Get<ui32>(); + NUdf::IBoxedValuePtr boxed(new TBrokenTupleBoxedValue<decltype(TUPLE_OF_PERSON_NO_LIST)>(TUPLE_OF_PERSON_NO_LIST, holePos)); + return NUdf::TUnboxedValuePod(std::move(boxed)); + } + + static const std::vector<NUdf::PersonStructWithOptList> LIST_OF_STRUCT_PERSON = { + STRUCT_PERSON_HITHCOCK_LIST, + STRUCT_PERSON_LOVECRAFT_LIST, + STRUCT_PERSON_KING_LIST + }; + + typedef NUdf::TDict<ui64,NUdf::PersonStructWithOptList> TIndexDictFromPersonList; + SIMPLE_UDF(TListOfPersonStructToIndexDict, TIndexDictFromPersonList(ui32)) { + Y_UNUSED(valueBuilder); + Y_UNUSED(args); + NUdf::IBoxedValuePtr boxed(new NUdf::TListRef<decltype(LIST_OF_STRUCT_PERSON)>(LIST_OF_STRUCT_PERSON)); + return valueBuilder->ToIndexDict(NUdf::TUnboxedValuePod(std::move(boxed))); + } + + SIMPLE_UDF(TListOfPersonStruct, NUdf::TListType<NUdf::PersonStructWithOptList>(ui32)) { + Y_UNUSED(valueBuilder); + Y_UNUSED(args); + NUdf::IBoxedValuePtr boxed(new NUdf::TListRef<decltype(LIST_OF_STRUCT_PERSON)>(LIST_OF_STRUCT_PERSON)); + return NUdf::TUnboxedValuePod(std::move(boxed)); + } + + SIMPLE_UDF(TListOfPersonStructWithBrokenIndexToDict, NUdf::TListType<NUdf::PersonStructWithOptList>()) { + Y_UNUSED(valueBuilder); + Y_UNUSED(args); + NUdf::IBoxedValuePtr boxed(new NUdf::TListRef<decltype(LIST_OF_STRUCT_PERSON), RAW_BROKEN_INDEX_LIST_TO_DICT>( + LIST_OF_STRUCT_PERSON)); + return NUdf::TUnboxedValuePod(std::move(boxed)); + } + + static const NUdf::PersonStruct* DICT_DIGIT2PERSON_BROKEN_CONTENT_BY_INDEX[] = { + &STRUCT_PERSON_HITHCOCK, &STRUCT_PERSON_JONNIE, &STRUCT_PERSON_LOVECRAFT + }; + const ui32 DICT_DIGIT2PERSON_BROKEN_PERSON_INDEX = 1; + const ui32 DICT_DIGIT2PERSON_BROKEN_STRUCT_INDEX = 2; + + const std::vector<std::pair<ui32, NUdf::IBoxedValuePtr>> MAKE_DICT_DIGIT2PERSON_BROKEN() { + std::vector<std::pair<ui32, NUdf::IBoxedValuePtr>> DICT_DIGIT2PERSON_BROKEN = { + { 333, new TBrokenStructBoxedValue<NUdf::PersonStruct>(STRUCT_PERSON_HITHCOCK, RAW_INDEX_NO_HOLE) }, + { 5, new TBrokenStructBoxedValue<NUdf::PersonStruct>(STRUCT_PERSON_JONNIE, DICT_DIGIT2PERSON_BROKEN_STRUCT_INDEX) }, + { 77, new TBrokenStructBoxedValue<NUdf::PersonStruct>(STRUCT_PERSON_LOVECRAFT, RAW_INDEX_NO_HOLE) }, + }; + + return DICT_DIGIT2PERSON_BROKEN; + } + + typedef NUdf::TDict<ui32,NUdf::PersonStruct> NUdfDictDigPerson; + + std::vector<std::pair<ui32, NUdf::IBoxedValuePtr>> MAKE_DICT_DIGIT2PERSON() { + const std::vector<std::pair<ui32, NUdf::IBoxedValuePtr>> DICT_DIGIT2PERSON = { + { 333, new TBrokenStructBoxedValue<NUdf::PersonStruct>(STRUCT_PERSON_HITHCOCK, RAW_INDEX_NO_HOLE) }, + { 5, new TBrokenStructBoxedValue<NUdf::PersonStruct>(STRUCT_PERSON_JONNIE, RAW_INDEX_NO_HOLE) }, + { 77, new TBrokenStructBoxedValue<NUdf::PersonStruct>(STRUCT_PERSON_LOVECRAFT, RAW_INDEX_NO_HOLE) }, + }; + + return DICT_DIGIT2PERSON; + } + + SIMPLE_UDF(TDictOfPerson, NUdfDictDigPerson()) { + Y_UNUSED(args); + Y_UNUSED(valueBuilder); + NUdf::IBoxedValuePtr boxed(new TBrokenDictBoxedValue<ui32, NUdf::IBoxedValuePtr>( + MAKE_DICT_DIGIT2PERSON(), std::make_pair(RAW_INDEX_NO_HOLE, RAW_INDEX_NO_HOLE))); + return NUdf::TUnboxedValuePod(std::move(boxed)); + } + + SIMPLE_UDF_RUN(TDictOfPersonBroken, NUdfDictDigPerson(), NUdf::TOptional<void>) { + Y_UNUSED(args); + Y_UNUSED(valueBuilder); + NUdf::IBoxedValuePtr boxed(new TBrokenDictBoxedValue<ui32, NUdf::IBoxedValuePtr>( + MAKE_DICT_DIGIT2PERSON_BROKEN(), std::make_pair(RAW_INDEX_NO_HOLE, RAW_INDEX_NO_HOLE))); + return NUdf::TUnboxedValuePod(std::move(boxed)); + } + + SIMPLE_MODULE(TUtUDF, + TException, + TVoid, + TNonEmpty, + TOptionalNonEmpty, + TOptionalEmpty, + TSub2Mul2BrokenOnLess2, + TBackSub2Mul2, + TSeqList, + TSeqListWithHole, + TTuple, + TDictDigDig, + TDictDigDigHoleAsOpt, + TPersonStruct, + TTupleOfPersonStruct, + TTupleOfPersonStructNoList, + TListOfPersonStructToIndexDict, + TListOfPersonStruct, + TListOfPersonStructWithBrokenIndexToDict, + TDictOfPerson, + TDictOfPersonBroken + ) +} // unnamed namespace + +TIntrusivePtr<IFunctionRegistry> CreateFunctionRegistryWithUDFs() { + auto freg = CreateFunctionRegistry(CreateBuiltinRegistry())->Clone(); + freg->AddModule("", "UtUDF", new TUtUDF()); + return freg; +} + +Y_UNIT_TEST_SUITE(TMiniKQLValidateTest) { + typedef std::function<std::vector<TRuntimeNode>(TProgramBuilder&)> BuildArgsFunc; + typedef std::function<void(const NUdf::TUnboxedValuePod&, const NUdf::IValueBuilder*)> ValidateValueFunc; + typedef std::function<void(const NUdf::TUnboxedValuePod&, const NUdf::IValueBuilder*, const TType* type)> FullValidateValueFunc; + + + void ProcessSimpleUdfFunc(const char* udfFuncName, BuildArgsFunc argsFunc = BuildArgsFunc(), ValidateValueFunc validateFunc = ValidateValueFunc(), + FullValidateValueFunc fullValidateFunc = FullValidateValueFunc(), + NUdf::EValidateMode validateMode = NUdf::EValidateMode::Lazy) { + TScopedAlloc alloc(__LOCATION__); + TTypeEnvironment env(alloc); + NUdf::ITypeInfoHelper::TPtr typeInfoHelper(new TTypeInfoHelper); + auto functionRegistry = CreateFunctionRegistryWithUDFs(); + auto randomProvider = CreateDeterministicRandomProvider(1); + auto timeProvider = CreateDeterministicTimeProvider(10000000); + TProgramBuilder pgmBuilder(env, *functionRegistry); + + auto funcName = pgmBuilder.Udf(udfFuncName); + std::vector<TRuntimeNode> execArgs; + if (argsFunc) { + execArgs = argsFunc(pgmBuilder); + } + auto pgmReturn = pgmBuilder.Apply(funcName, execArgs); + + TExploringNodeVisitor explorer; + explorer.Walk(pgmReturn.GetNode(), env); + TComputationPatternOpts opts(alloc.Ref(), env, GetBuiltinFactory(), + functionRegistry.Get(), validateMode, + NUdf::EValidatePolicy::Exception, "OFF", EGraphPerProcess::Multi); + auto pattern = MakeComputationPattern(explorer, pgmReturn, {}, opts); + auto graph = pattern->Clone(opts.ToComputationOptions(*randomProvider, *timeProvider)); + const auto builder = static_cast<TDefaultValueBuilder*>(graph->GetTerminator()); + builder->RethrowAtTerminate(); + const TBindTerminator bind(graph->GetTerminator()); + auto value = graph->GetValue(); + + if (validateFunc) { + validateFunc(value, builder); + } + if (fullValidateFunc) { + ui32 flags = 0; + TFunctionTypeInfo funcInfo; + TType* userType = nullptr; + TStringBuf typeConfig; + TStatus status = functionRegistry->FindFunctionTypeInfo(env, typeInfoHelper, nullptr, udfFuncName, userType, typeConfig, flags, {}, nullptr, &funcInfo); + MKQL_ENSURE(status.IsOk(), status.GetError()); + auto type = funcInfo.FunctionType->GetReturnType(); + fullValidateFunc(value, builder, type); + } + } + + Y_UNIT_TEST(TestUdfException) { + ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) { + valueBuilder->NewStringNotFilled(0xBAD).AsStringValue().Ref(); // Leak string. + NUdf::TBoxedValueAccessor::Skip(*value.GetListIterator().AsBoxed().Release()); // Leak value and throw exception. + }; + UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.Exception", {}, validateFunc), yexception); + UNIT_ASSERT_VALUES_EQUAL(TThrowerValue::Count, 0L); + } + + Y_UNIT_TEST(TestUdfResultCheckVoid) { + ProcessSimpleUdfFunc("UtUDF.Void"); + } + + Y_UNIT_TEST(TestUdfResultCheckExceptionOnEmpty) { + TScopedAlloc alloc(__LOCATION__); + TTypeEnvironment env(alloc); + bool wrapped = false; + UNIT_ASSERT_EXCEPTION(TValidate<TValidateErrorPolicyThrow>::Value(nullptr, env.GetTypeOfType(), + NUdf::TUnboxedValuePod(), "ut for verify empty value exception", &wrapped), TUdfValidateException); + UNIT_ASSERT(!wrapped); + } + + Y_UNIT_TEST(TestUdfResultCheckNonEmpty) { + ProcessSimpleUdfFunc("UtUDF.NonEmpty"); + } + + Y_UNIT_TEST(TestUdfResultCheckOptionalNonEmpty) { + ProcessSimpleUdfFunc("UtUDF.OptionalNonEmpty"); + } + + Y_UNIT_TEST(TestUdfResultCheckOptionalEmpty) { + ProcessSimpleUdfFunc("UtUDF.OptionalEmpty"); + } + + std::vector<TRuntimeNode> MakeCallableInArgs(ui32 testVal, TProgramBuilder& pgmBuilder) { + const auto& functionRegistry = pgmBuilder.GetFunctionRegistry(); + + const auto udfFuncName = "UtUDF.Sub2Mul2BrokenOnLess2"; + ui32 flags = 0; + TFunctionTypeInfo funcInfo; + TType* userType = nullptr; + TStringBuf typeConfig; + NUdf::ITypeInfoHelper::TPtr typeInfoHelper(new TTypeInfoHelper); + TStatus status = functionRegistry.FindFunctionTypeInfo(pgmBuilder.GetTypeEnvironment(), typeInfoHelper, nullptr, + udfFuncName, userType, typeConfig, flags, {}, nullptr, &funcInfo); + MKQL_ENSURE(status.IsOk(), status.GetError()); + auto callable = pgmBuilder.Udf(udfFuncName); + return std::vector<TRuntimeNode>{callable, pgmBuilder.NewDataLiteral(testVal)}; + }; + + Y_UNIT_TEST(TestVerifyArgsCallableCorrect) { + ui32 testVal = 44; + BuildArgsFunc argsFunc = [testVal](TProgramBuilder& pgmBuilder) { + return MakeCallableInArgs(testVal, pgmBuilder); + }; + ValidateValueFunc validateFunc = [testVal](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) { + Y_UNUSED(valueBuilder); + UNIT_ASSERT_VALUES_EQUAL(testVal, value.Get<ui32>()); + }; + ProcessSimpleUdfFunc("UtUDF.BackSub2Mul2", argsFunc, validateFunc); + } + + Y_UNIT_TEST(TestVerifyArgsCallableBrokenOnArgument) { + ui32 testVal = 101; + BuildArgsFunc argsFunc = [testVal](TProgramBuilder& pgmBuilder) { + return MakeCallableInArgs(testVal, pgmBuilder); + }; + UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.BackSub2Mul2", argsFunc), TUdfValidateException); + } + + Y_UNIT_TEST(TestVerifyArgsCallableBrokenOnReturn) { + ui32 testVal = 1; + BuildArgsFunc argsFunc = [testVal](TProgramBuilder& pgmBuilder) { + return MakeCallableInArgs(testVal, pgmBuilder); + }; + UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.BackSub2Mul2", argsFunc), TUdfValidateException); + } + + Y_UNIT_TEST(TestUdfResultCheckEmptySeqList) { + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral<ui32>(0)}; + }; + ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) { + Y_UNUSED(valueBuilder); + auto listIter = value.GetListIterator(); + UNIT_ASSERT(!listIter.Skip()); + }; + ProcessSimpleUdfFunc("UtUDF.SeqList", argsFunc, validateFunc); + } + + Y_UNIT_TEST(TestUdfResultCheckSeqList) { + static constexpr ui32 listSize = 31; + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(listSize)}; + }; + ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) { + Y_UNUSED(valueBuilder); + ui32 index = 0; + auto listIter = value.GetListIterator(); + for (NUdf::TUnboxedValue item; listIter.Next(item); ++index) { + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), index); + } + UNIT_ASSERT_VALUES_EQUAL(index, listSize); + }; + ProcessSimpleUdfFunc("UtUDF.SeqList", argsFunc, validateFunc); + } + + Y_UNIT_TEST(TestUdfResultCheckSeqListWithHoleFirst) { + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + const ui32 listSize = 31; + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(listSize), + pgmBuilder.NewDataLiteral<ui32>(0)}; + }; + ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) { + Y_UNUSED(valueBuilder); + auto listIter = value.GetListIterator(); + NUdf::TUnboxedValue item; + UNIT_ASSERT_EXCEPTION(listIter.Next(item), TUdfValidateException); + for (ui32 index = 1; listIter.Next(item); ++index) { + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), index); + } + }; + ProcessSimpleUdfFunc("UtUDF.SeqListWithHole", argsFunc, validateFunc); + UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.SeqListWithHole", argsFunc, validateFunc, {}, NUdf::EValidateMode::Greedy), TUdfValidateException); + } + + Y_UNIT_TEST(TestUdfResultCheckSeqListWithHoleMiddle) { + static constexpr ui32 listSize = 31; + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(listSize), + pgmBuilder.NewDataLiteral(listSize / 2)}; + }; + ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) { + Y_UNUSED(valueBuilder); + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), listSize); + ui32 index = 0; + const auto listIter = value.GetListIterator(); + for (NUdf::TUnboxedValue item; index < listSize / 2 && listIter.Next(item); ++index) { + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), index); + } + NUdf::TUnboxedValue bad; + UNIT_ASSERT_EXCEPTION(listIter.Next(bad), TUdfValidateException); + ++index; + for (NUdf::TUnboxedValue item; listIter.Next(item); ++index) { + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), index); + } + }; + ProcessSimpleUdfFunc("UtUDF.SeqListWithHole", argsFunc, validateFunc); + UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.SeqListWithHole", argsFunc, validateFunc, {}, NUdf::EValidateMode::Greedy), TUdfValidateException); + } + + Y_UNIT_TEST(TestUdfResultCheckSeqListWithHoleLast) { + static constexpr ui32 listSize = 31; + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(listSize), + pgmBuilder.NewDataLiteral(listSize - 1)}; + }; + ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) { + Y_UNUSED(valueBuilder); + UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), listSize); + ui32 index = 0; + auto listIter = value.GetListIterator(); + for (NUdf::TUnboxedValue item; index < listSize - 1 && listIter.Next(item); ++index) { + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), index); + } + UNIT_ASSERT_VALUES_EQUAL(index, listSize - 1); + NUdf::TUnboxedValue bad; + UNIT_ASSERT_EXCEPTION(listIter.Next(bad), TUdfValidateException); + }; + ProcessSimpleUdfFunc("UtUDF.SeqListWithHole", argsFunc, validateFunc); + UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.SeqListWithHole", argsFunc, validateFunc, {}, NUdf::EValidateMode::Greedy), TUdfValidateException); + } + + Y_UNIT_TEST(TestUdfResultCheckTuple) { + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE)}; + }; + ProcessSimpleUdfFunc("UtUDF.Tuple", argsFunc); + } + + Y_UNIT_TEST(TestUdfResultCheckTupleWithHoleFirst) { + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral<ui32>(0)}; + }; + UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.Tuple", argsFunc), TUdfValidateException); + } + + Y_UNIT_TEST(TestUdfResultCheckTupleWithHoleMiddle) { + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral<ui32>(std::tuple_size<decltype(TUPLE)>::value / 2)}; + }; + UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.Tuple", argsFunc), TUdfValidateException); + } + + Y_UNIT_TEST(TestUdfResultCheckTupleWithHoleLast) { + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral<ui32>(std::tuple_size<decltype(TUPLE)>::value - 1)}; + }; + UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.Tuple", argsFunc), TUdfValidateException); + } + + Y_UNIT_TEST(TestUdfResultCheckDictDigitDigitFull) { + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE), + pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE)}; + }; + ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) { + Y_UNUSED(valueBuilder); + auto dictIter = value.GetDictIterator(); + ui32 index = 0; + for (NUdf::TUnboxedValue key, payload; dictIter.NextPair(key, payload); ++index) { + UNIT_ASSERT_VALUES_EQUAL(key.Get<ui32>(), DICT_DIGIT2DIGIT[index].first); + UNIT_ASSERT_VALUES_EQUAL(payload.Get<ui64>(), DICT_DIGIT2DIGIT[index].second); + } + UNIT_ASSERT_VALUES_EQUAL(index, DICT_DIGIT2DIGIT.size()); + }; + ProcessSimpleUdfFunc("UtUDF.DictDigDig", argsFunc, validateFunc); + } + + Y_UNIT_TEST(TestUdfResultCheckDictDigitDigitKeyHole) { + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral<ui32>(0), + pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE)}; + }; + ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) { + Y_UNUSED(valueBuilder); + auto dictIter = value.GetDictIterator(); + NUdf::TUnboxedValue key, payload; + UNIT_ASSERT_EXCEPTION(dictIter.NextPair(key, payload), TUdfValidateException); + for (ui32 index = 1; dictIter.NextPair(key, payload); ++index) { + UNIT_ASSERT_VALUES_EQUAL(key.Get<ui32>(), DICT_DIGIT2DIGIT[index].first); + UNIT_ASSERT_VALUES_EQUAL(payload.Get<ui64>(), DICT_DIGIT2DIGIT[index].second); + } + }; + ProcessSimpleUdfFunc("UtUDF.DictDigDig", argsFunc, validateFunc); + } + + Y_UNIT_TEST(TestUdfResultCheckDictDigitDigitValueHole) { + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE), + pgmBuilder.NewDataLiteral<ui32>(DICT_DIGIT2DIGIT.size() - 1)}; + }; + ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) { + Y_UNUSED(valueBuilder); + auto dictIter = value.GetDictIterator(); + NUdf::TUnboxedValue key, payload; + for (ui32 index = 0; index < DICT_DIGIT2DIGIT.size() - 1 && dictIter.NextPair(key, payload); ++index) { + UNIT_ASSERT_VALUES_EQUAL(key.Get<ui32>(), DICT_DIGIT2DIGIT[index].first); + UNIT_ASSERT_VALUES_EQUAL(payload.Get<ui64>(), DICT_DIGIT2DIGIT[index].second); + } + UNIT_ASSERT_EXCEPTION(dictIter.NextPair(key, payload), TUdfValidateException); + }; + ProcessSimpleUdfFunc("UtUDF.DictDigDig", argsFunc, validateFunc); + } + Y_UNIT_TEST(TestUdfResultCheckDictDigitDigitHoleAsOptKeyHole) { + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE), + pgmBuilder.NewDataLiteral<ui32>(0)}; + }; + ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) { + Y_UNUSED(valueBuilder); + auto dictIter = value.GetDictIterator(); + NUdf::TUnboxedValue key, payload; + UNIT_ASSERT_EXCEPTION(dictIter.NextPair(key, payload), TUdfValidateException); + for (ui32 index = 1; dictIter.NextPair(key, payload); ++index) { + UNIT_ASSERT_VALUES_EQUAL(key.Get<ui32>(), DICT_DIGIT2DIGIT[index].first); + UNIT_ASSERT_VALUES_EQUAL(payload.Get<ui64>(), DICT_DIGIT2DIGIT[index].second); + } + }; + ProcessSimpleUdfFunc("UtUDF.DictDigDig", argsFunc, validateFunc); + } + + Y_UNIT_TEST(TestUdfResultCheckDictDigitDigitHoleAsOptValueHole) { + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral<ui32>(DICT_DIGIT2DIGIT.size() - 1), + pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE)}; + }; + ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) { + Y_UNUSED(valueBuilder); + auto dictIter = value.GetDictIterator(); + NUdf::TUnboxedValue key, payload; + for (ui32 index = 0; index < DICT_DIGIT2DIGIT.size() - 1 && dictIter.NextPair(key, payload); ++index) { + UNIT_ASSERT_VALUES_EQUAL(key.Get<ui32>(), DICT_DIGIT2DIGIT[index].first); + UNIT_ASSERT_VALUES_EQUAL(payload.Get<ui64>(), DICT_DIGIT2DIGIT[index].second); + } + UNIT_ASSERT_EXCEPTION(dictIter.NextPair(key, payload), TUdfValidateException); + }; + ProcessSimpleUdfFunc("UtUDF.DictDigDig", argsFunc, validateFunc); + } + + Y_UNIT_TEST(TestUdfResultCheckPersonStruct) { + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE)}; + }; + ProcessSimpleUdfFunc("UtUDF.PersonStruct", argsFunc); + } + + Y_UNIT_TEST(TestUdfResultCheckPersonStructWithHoleFirst) { + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral<ui32>(0)}; + }; + UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.PersonStruct", argsFunc), TUdfValidateException); + } + + Y_UNIT_TEST(TestUdfResultCheckPersonStructWithHoleMiddle) { + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral<ui32>(1)}; + }; + UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.PersonStruct", argsFunc), TUdfValidateException); + } + + Y_UNIT_TEST(TestUdfResultCheckPersonStructWithHoleLast) { + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral<ui32>(2)}; + }; + UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.PersonStruct", argsFunc), TUdfValidateException); + } + + Y_UNIT_TEST(TestUdfResultCheckTupleOfPersonStruct) { + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE)}; + }; + FullValidateValueFunc fullValidateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder, + const TType* type) { + bool wrapped = false; + TValidate<TValidateErrorPolicyThrow, TValidateModeGreedy<TValidateErrorPolicyThrow>>::Value(valueBuilder, type, NUdf::TUnboxedValuePod(value), "full verify func", &wrapped); + UNIT_ASSERT(!wrapped); + TValidate<TValidateErrorPolicyThrow, TValidateModeLazy<TValidateErrorPolicyThrow>>::Value(valueBuilder, type, NUdf::TUnboxedValuePod(value), "full verify func", &wrapped); + UNIT_ASSERT(wrapped); + }; + ProcessSimpleUdfFunc("UtUDF.TupleOfPersonStruct", argsFunc, {}, fullValidateFunc); + ProcessSimpleUdfFunc("UtUDF.TupleOfPersonStruct", argsFunc, {}, fullValidateFunc, NUdf::EValidateMode::Greedy); + } + + Y_UNIT_TEST(TestUdfResultCheckTupleOfPersonStructNoList) { + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE)}; + }; + FullValidateValueFunc fullValidateFunc = [](const NUdf::TUnboxedValuePod& value, + const NUdf::IValueBuilder* valueBuilder, const TType* type) { + bool wrapped = false; + TValidate<TValidateErrorPolicyThrow>::Value(valueBuilder, type, NUdf::TUnboxedValuePod(value), "full verify func", &wrapped); + UNIT_ASSERT(!wrapped); + }; + ProcessSimpleUdfFunc("UtUDF.TupleOfPersonStructNoList", argsFunc, {}, fullValidateFunc); + ProcessSimpleUdfFunc("UtUDF.TupleOfPersonStructNoList", argsFunc, {}, fullValidateFunc, NUdf::EValidateMode::Greedy); + } + + void ValidateDictOfPersonStructFunc(const NUdf::TUnboxedValuePod& value, ui32 lookupIndex = 2, ui32 broken_index = RAW_INDEX_NO_HOLE) { + const auto person = value.Lookup(NUdf::TUnboxedValuePod(ui64(lookupIndex))); + UNIT_ASSERT(person); + auto firstName = person.GetElement(NUdf::PersonStructWithOptList::MetaIndexes[0]); + UNIT_ASSERT_VALUES_EQUAL(TString(firstName.AsStringRef()), LIST_OF_STRUCT_PERSON[lookupIndex].FirstName); + auto lastName = person.GetElement(NUdf::PersonStructWithOptList::MetaIndexes[1]); + UNIT_ASSERT_VALUES_EQUAL(TString(lastName.AsStringRef()), LIST_OF_STRUCT_PERSON[lookupIndex].LastName); + UNIT_ASSERT_VALUES_EQUAL(person.GetElement(NUdf::PersonStructWithOptList::MetaIndexes[2]).Get<ui32>(), LIST_OF_STRUCT_PERSON[lookupIndex].Age); + UNIT_ASSERT(!person.GetElement(NUdf::PersonStructWithOptList::MetaIndexes[3])); + auto dictIter = value.GetDictIterator(); + NUdf::TUnboxedValue key, payload; + for (ui32 index = 0; index < broken_index && dictIter.NextPair(key, payload); ++index) { + UNIT_ASSERT_VALUES_EQUAL(key.Get<ui64>(), index); + auto person = payload; + auto firstName = person.GetElement(NUdf::PersonStructWithOptList::MetaIndexes[0]); + UNIT_ASSERT_VALUES_EQUAL(TString(firstName.AsStringRef()), LIST_OF_STRUCT_PERSON[index].FirstName); + auto lastName = person.GetElement(NUdf::PersonStructWithOptList::MetaIndexes[1]); + UNIT_ASSERT_VALUES_EQUAL(TString(lastName.AsStringRef()), LIST_OF_STRUCT_PERSON[index].LastName); + UNIT_ASSERT_VALUES_EQUAL(person.GetElement(NUdf::PersonStructWithOptList::MetaIndexes[2]).Get<ui32>(), LIST_OF_STRUCT_PERSON[index].Age); + const auto origListPtr = LIST_OF_STRUCT_PERSON[index].Tags; + if (origListPtr.empty()) { + UNIT_ASSERT(!person.GetElement(NUdf::PersonStructWithOptList::MetaIndexes[3])); + } else { + auto memberTags = person.GetElement(NUdf::PersonStructWithOptList::MetaIndexes[3]); + UNIT_ASSERT(memberTags); + UNIT_ASSERT_VALUES_EQUAL(memberTags.GetListLength(), origListPtr.size()); + auto origIter = origListPtr.begin(); + auto iter = memberTags.GetListIterator(); + for (NUdf::TUnboxedValue item; iter.Next(item); ++origIter) { + UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), *origIter); + } + } + } + if (broken_index < RAW_INDEX_NO_HOLE) + UNIT_ASSERT_EXCEPTION(dictIter.NextPair(key, payload), TUdfValidateException); + } + + Y_UNIT_TEST(TestUdfResultCheckListOfPersonStructToIndexDict) { + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE)}; + }; + ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) { + Y_UNUSED(valueBuilder); + ValidateDictOfPersonStructFunc(value); + }; + ProcessSimpleUdfFunc("UtUDF.ListOfPersonStructToIndexDict", argsFunc, validateFunc); + } + + Y_UNIT_TEST(TestUdfResultCheckListOfPersonStruct) { + BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) { + return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE)}; + }; + FullValidateValueFunc fullValidateFunc = [](const NUdf::TUnboxedValuePod& value, + const NUdf::IValueBuilder* valueBuilder, const TType* type) { + Y_UNUSED(type); + auto indexDict = valueBuilder->ToIndexDict(value); + ValidateDictOfPersonStructFunc(indexDict); + }; + ProcessSimpleUdfFunc("UtUDF.ListOfPersonStruct", argsFunc, {}, fullValidateFunc); + } + + Y_UNIT_TEST(TestUdfResultCheckListOfPersonStructWithBrokenIndexToDict) { + FullValidateValueFunc fullValidateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder, + const TType* type) { + Y_UNUSED(type); + auto indexDict = valueBuilder->ToIndexDict(value); + static_assert(RAW_BROKEN_INDEX_LIST_TO_DICT == 1, "a list is too small"); + ValidateDictOfPersonStructFunc(indexDict, RAW_BROKEN_INDEX_LIST_TO_DICT - 1, RAW_BROKEN_INDEX_LIST_TO_DICT); + /// verify lookup fail on broken index + UNIT_ASSERT_EXCEPTION(ValidateDictOfPersonStructFunc(indexDict, RAW_BROKEN_INDEX_LIST_TO_DICT, RAW_BROKEN_INDEX_LIST_TO_DICT), TUdfValidateException); + ValidateDictOfPersonStructFunc(indexDict, RAW_BROKEN_INDEX_LIST_TO_DICT + 1, RAW_BROKEN_INDEX_LIST_TO_DICT); + }; + ProcessSimpleUdfFunc("UtUDF.ListOfPersonStructWithBrokenIndexToDict", {}, {}, fullValidateFunc); + } + + Y_UNIT_TEST(TestUdfResultCheckDictOfPerson) { + ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder*) { + auto dictIter = value.GetDictIterator(); + NUdf::TUnboxedValue key, payload; + for (ui32 index = 0; dictIter.NextPair(key, payload); ++index) { + UNIT_ASSERT_VALUES_EQUAL(key.Get<ui32>(), MAKE_DICT_DIGIT2PERSON()[index].first); + auto person = payload; + auto firstName = person.GetElement(NUdf::PersonStruct::MetaIndexes[0]); + UNIT_ASSERT_VALUES_EQUAL(TString(firstName.AsStringRef()), DICT_DIGIT2PERSON_BROKEN_CONTENT_BY_INDEX[index]->FirstName); + auto lastName = person.GetElement(NUdf::PersonStruct::MetaIndexes[1]); + UNIT_ASSERT_VALUES_EQUAL(TString(lastName.AsStringRef()), DICT_DIGIT2PERSON_BROKEN_CONTENT_BY_INDEX[index]->LastName); + UNIT_ASSERT_VALUES_EQUAL(person.GetElement(NUdf::PersonStruct::MetaIndexes[2]).Get<ui32>(), DICT_DIGIT2PERSON_BROKEN_CONTENT_BY_INDEX[index]->Age); + } + }; + ProcessSimpleUdfFunc("UtUDF.DictOfPerson", {}, validateFunc); + } + + Y_UNIT_TEST(TestUdfResultCheckDictOfPersonBroken) { + ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder*) { + auto dictIter = value.GetDictIterator(); + NUdf::TUnboxedValue key, payload; + for (ui32 index = 0; index < DICT_DIGIT2PERSON_BROKEN_PERSON_INDEX && dictIter.NextPair(key, payload); ++index) { + UNIT_ASSERT_VALUES_EQUAL(key.Get<ui32>(), MAKE_DICT_DIGIT2PERSON_BROKEN()[index].first); + auto person = payload; + auto firstName = person.GetElement(NUdf::PersonStruct::MetaIndexes[0]); + UNIT_ASSERT_VALUES_EQUAL(TString(firstName.AsStringRef()), DICT_DIGIT2PERSON_BROKEN_CONTENT_BY_INDEX[index]->FirstName); + auto lastName = person.GetElement(NUdf::PersonStruct::MetaIndexes[1]); + UNIT_ASSERT_VALUES_EQUAL(TString(lastName.AsStringRef()), DICT_DIGIT2PERSON_BROKEN_CONTENT_BY_INDEX[index]->LastName); + UNIT_ASSERT_VALUES_EQUAL(person.GetElement(NUdf::PersonStruct::MetaIndexes[2]).Get<ui32>(), DICT_DIGIT2PERSON_BROKEN_CONTENT_BY_INDEX[index]->Age); + } + UNIT_ASSERT_EXCEPTION(dictIter.NextPair(key, payload), TUdfValidateException); + }; + ProcessSimpleUdfFunc("UtUDF.DictOfPersonBroken", {}, validateFunc); + } + +} + +} diff --git a/ydb/library/yql/minikql/computation/mkql_value_builder_ut.cpp b/ydb/library/yql/minikql/computation/mkql_value_builder_ut.cpp new file mode 100644 index 00000000000..46a04cc001c --- /dev/null +++ b/ydb/library/yql/minikql/computation/mkql_value_builder_ut.cpp @@ -0,0 +1,373 @@ +#include "mkql_value_builder.h" +#include "mkql_computation_node_holders.h" + +#include <ydb/library/yql/minikql/mkql_function_registry.h> +#include <ydb/library/yql/minikql/mkql_type_builder.h> +#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> +#include <ydb/library/yql/parser/pg_catalog/catalog.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <arrow/array/builder_primitive.h> +#include <arrow/c/abi.h> +#include <arrow/scalar.h> +#include <arrow/chunked_array.h> + +namespace NYql { +namespace NCommon { + +TString PgValueToNativeText(const NUdf::TUnboxedValuePod& value, ui32 pgTypeId); +TString PgValueToNativeBinary(const NUdf::TUnboxedValuePod& value, ui32 pgTypeId); + +} +} + +namespace NKikimr { + +using namespace NUdf; +using namespace NYql::NCommon; + +namespace NMiniKQL { + +namespace { + TString AsString(const TStringValue& v) { + return { v.Data(), v.Size() }; + } +} + +class TMiniKQLValueBuilderTest: public TTestBase { +public: + TMiniKQLValueBuilderTest() + : FunctionRegistry(CreateFunctionRegistry(CreateBuiltinRegistry())) + , Alloc(__LOCATION__) + , Env(Alloc) + , MemInfo("Memory") + , HolderFactory(Alloc.Ref(), MemInfo, FunctionRegistry.Get()) + , Builder(HolderFactory, NUdf::EValidatePolicy::Exception) + , TypeInfoHelper(new TTypeInfoHelper()) + , FunctionTypeInfoBuilder(Env, TypeInfoHelper, "", nullptr, {}) + { + BoolOid = NYql::NPg::LookupType("bool").TypeId; + } + + const IPgBuilder& GetPgBuilder() const { + return Builder.GetPgBuilder(); + } + +private: + TIntrusivePtr<NMiniKQL::IFunctionRegistry> FunctionRegistry; + TScopedAlloc Alloc; + TTypeEnvironment Env; + TMemoryUsageInfo MemInfo; + THolderFactory HolderFactory; + TDefaultValueBuilder Builder; + NUdf::ITypeInfoHelper::TPtr TypeInfoHelper; + TFunctionTypeInfoBuilder FunctionTypeInfoBuilder; + ui32 BoolOid = 0; + + UNIT_TEST_SUITE(TMiniKQLValueBuilderTest); + UNIT_TEST(TestEmbeddedVariant); + UNIT_TEST(TestBoxedVariant); + UNIT_TEST(TestSubstring); + UNIT_TEST(TestPgValueFromErrors); + UNIT_TEST(TestPgValueFromText); + UNIT_TEST(TestPgValueFromBinary); + UNIT_TEST(TestConvertToFromPg); + UNIT_TEST(TestConvertToFromPgNulls); + UNIT_TEST(TestPgNewString); + UNIT_TEST(TestArrowBlock); + UNIT_TEST_SUITE_END(); + + + void TestEmbeddedVariant() { + const auto v = Builder.NewVariant(62, TUnboxedValuePod((ui64) 42)); + UNIT_ASSERT(v); + UNIT_ASSERT(!v.IsBoxed()); + UNIT_ASSERT_VALUES_EQUAL(62, v.GetVariantIndex()); + UNIT_ASSERT_VALUES_EQUAL(42, v.GetVariantItem().Get<ui64>()); + } + + void TestBoxedVariant() { + const auto v = Builder.NewVariant(63, TUnboxedValuePod((ui64) 42)); + UNIT_ASSERT(v); + UNIT_ASSERT(v.IsBoxed()); + UNIT_ASSERT_VALUES_EQUAL(63, v.GetVariantIndex()); + UNIT_ASSERT_VALUES_EQUAL(42, v.GetVariantItem().Get<ui64>()); + } + + void TestSubstring() { + const auto string = Builder.NewString("0123456789qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM"); + UNIT_ASSERT(string); + + const auto zero = Builder.SubString(string, 7, 0); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(""), TStringBuf(zero.AsStringRef())); + + const auto tail = Builder.SubString(string, 60, 8); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf("NM"), TStringBuf(tail.AsStringRef())); + + const auto small = Builder.SubString(string, 2, 14); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf("23456789qwerty"), TStringBuf(small.AsStringRef())); + + const auto one = Builder.SubString(string, 3, 15); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf("3456789qwertyui"), TStringBuf(one.AsStringRef())); + UNIT_ASSERT_VALUES_EQUAL(string.AsStringValue().Data(), one.AsStringValue().Data()); + + const auto two = Builder.SubString(string, 10, 30); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf("qwertyuiopasdfghjklzxcvbnmQWER"), TStringBuf(two.AsStringRef())); + UNIT_ASSERT_VALUES_EQUAL(string.AsStringValue().Data(), two.AsStringValue().Data()); + } + + void TestPgValueFromErrors() { + const TBindTerminator bind(&Builder); // to raise exception instead of abort + { + TStringValue error(""); + auto r = GetPgBuilder().ValueFromText(BoolOid, "", error); + UNIT_ASSERT(!r); + UNIT_ASSERT_STRING_CONTAINS(AsString(error), "ERROR: invalid input syntax for type boolean: \"\""); + } + + { + TStringValue error(""); + auto r = GetPgBuilder().ValueFromText(BoolOid, "zzzz", error); + UNIT_ASSERT(!r); + UNIT_ASSERT_STRING_CONTAINS(AsString(error), "ERROR: invalid input syntax for type boolean: \"zzzz\""); + } + + { + TStringValue error(""); + auto r = GetPgBuilder().ValueFromBinary(BoolOid, "", error); + UNIT_ASSERT(!r); + UNIT_ASSERT_STRING_CONTAINS(AsString(error), "ERROR: no data left in message"); + } + + { + TStringValue error(""); + auto r = GetPgBuilder().ValueFromBinary(BoolOid, "zzzz", error); + UNIT_ASSERT(!r); + UNIT_ASSERT_STRING_CONTAINS(AsString(error), "Not all data has been consumed by 'recv' function: boolrecv, data size: 4, consumed size: 1"); + } + } + + void TestPgValueFromText() { + const TBindTerminator bind(&Builder); + for (auto validTrue : { "t"sv, "true"sv }) { + TStringValue error(""); + auto r = GetPgBuilder().ValueFromText(BoolOid, validTrue, error); + UNIT_ASSERT(r); + UNIT_ASSERT_VALUES_EQUAL(AsString(error), ""); + auto s = PgValueToNativeText(r, BoolOid); + UNIT_ASSERT_VALUES_EQUAL(s, "t"); + } + + for (auto validFalse : { "f"sv, "false"sv }) { + TStringValue error(""); + auto r = GetPgBuilder().ValueFromText(BoolOid, validFalse, error); + UNIT_ASSERT(r); + UNIT_ASSERT_VALUES_EQUAL(AsString(error), ""); + auto s = PgValueToNativeText(r, BoolOid); + UNIT_ASSERT_VALUES_EQUAL(s, "f"); + } + } + + void TestPgValueFromBinary() { + const TBindTerminator bind(&Builder); + TStringValue error(""); + auto t = GetPgBuilder().ValueFromText(BoolOid, "true", error); + UNIT_ASSERT(t); + auto f = GetPgBuilder().ValueFromText(BoolOid, "false", error); + UNIT_ASSERT(f); + + auto ts = PgValueToNativeBinary(t, BoolOid); + auto fs = PgValueToNativeBinary(f, BoolOid); + { + auto r = GetPgBuilder().ValueFromBinary(BoolOid, ts, error); + UNIT_ASSERT(r); + auto s = PgValueToNativeText(r, BoolOid); + UNIT_ASSERT_VALUES_EQUAL(s, "t"); + } + + { + auto r = GetPgBuilder().ValueFromBinary(BoolOid, fs, error); + UNIT_ASSERT(r); + auto s = PgValueToNativeText(r, BoolOid); + UNIT_ASSERT_VALUES_EQUAL(s, "f"); + } + } + + void TestConvertToFromPg() { + const TBindTerminator bind(&Builder); + auto boolType = FunctionTypeInfoBuilder.SimpleType<bool>(); + { + auto v = GetPgBuilder().ConvertToPg(TUnboxedValuePod(true), boolType, BoolOid); + auto s = PgValueToNativeText(v, BoolOid); + UNIT_ASSERT_VALUES_EQUAL(s, "t"); + + auto from = GetPgBuilder().ConvertFromPg(v, BoolOid, boolType); + UNIT_ASSERT_VALUES_EQUAL(from.Get<bool>(), true); + } + + { + auto v = GetPgBuilder().ConvertToPg(TUnboxedValuePod(false), boolType, BoolOid); + auto s = PgValueToNativeText(v, BoolOid); + UNIT_ASSERT_VALUES_EQUAL(s, "f"); + + auto from = GetPgBuilder().ConvertFromPg(v, BoolOid, boolType); + UNIT_ASSERT_VALUES_EQUAL(from.Get<bool>(), false); + } + } + + void TestConvertToFromPgNulls() { + const TBindTerminator bind(&Builder); + auto boolOptionalType = FunctionTypeInfoBuilder.Optional()->Item<bool>().Build(); + + { + auto v = GetPgBuilder().ConvertToPg(TUnboxedValuePod(), boolOptionalType, BoolOid); + UNIT_ASSERT(!v); + } + + { + auto v = GetPgBuilder().ConvertFromPg(TUnboxedValuePod(), BoolOid, boolOptionalType); + UNIT_ASSERT(!v); + } + } + + void TestPgNewString() { + { + auto& pgText = NYql::NPg::LookupType("text"); + UNIT_ASSERT_VALUES_EQUAL(pgText.TypeLen, -1); + + auto s = GetPgBuilder().NewString(pgText.TypeLen, pgText.TypeId, "ABC"); + auto utf8Type = FunctionTypeInfoBuilder.SimpleType<TUtf8>(); + auto from = GetPgBuilder().ConvertFromPg(s, pgText.TypeId, utf8Type); + UNIT_ASSERT_VALUES_EQUAL((TStringBuf)from.AsStringRef(), "ABC"sv); + } + + { + auto& pgCString = NYql::NPg::LookupType("cstring"); + UNIT_ASSERT_VALUES_EQUAL(pgCString.TypeLen, -2); + + auto s = GetPgBuilder().NewString(pgCString.TypeLen, pgCString.TypeId, "ABC"); + auto utf8Type = FunctionTypeInfoBuilder.SimpleType<TUtf8>(); + auto from = GetPgBuilder().ConvertFromPg(s, pgCString.TypeId, utf8Type); + UNIT_ASSERT_VALUES_EQUAL((TStringBuf)from.AsStringRef(), "ABC"sv); + } + + { + auto& byteaString = NYql::NPg::LookupType("bytea"); + UNIT_ASSERT_VALUES_EQUAL(byteaString.TypeLen, -1); + + auto s = GetPgBuilder().NewString(byteaString.TypeLen, byteaString.TypeId, "ABC"); + auto stringType = FunctionTypeInfoBuilder.SimpleType<char*>(); + auto from = GetPgBuilder().ConvertFromPg(s, byteaString.TypeId, stringType); + UNIT_ASSERT_VALUES_EQUAL((TStringBuf)from.AsStringRef(), "ABC"sv); + } + } + + void TestArrowBlock() { + auto type = FunctionTypeInfoBuilder.SimpleType<ui64>(); + auto atype = TypeInfoHelper->MakeArrowType(type); + + { + arrow::Datum d1(std::make_shared<arrow::UInt64Scalar>(123)); + NUdf::TUnboxedValue val1 = HolderFactory.CreateArrowBlock(std::move(d1)); + bool isScalar; + ui64 length; + auto chunks = Builder.GetArrowBlockChunks(val1, isScalar, length); + UNIT_ASSERT_VALUES_EQUAL(chunks, 1); + UNIT_ASSERT(isScalar); + UNIT_ASSERT_VALUES_EQUAL(length, 1); + + ArrowArray arr1; + Builder.ExportArrowBlock(val1, 0, &arr1); + NUdf::TUnboxedValue val2 = Builder.ImportArrowBlock(&arr1, 1, isScalar, *atype); + const auto& d2 = TArrowBlock::From(val2).GetDatum(); + UNIT_ASSERT(d2.is_scalar()); + UNIT_ASSERT_VALUES_EQUAL(d2.scalar_as<arrow::UInt64Scalar>().value, 123); + } + + { + arrow::UInt64Builder builder; + UNIT_ASSERT(builder.Reserve(3).ok()); + builder.UnsafeAppend(ui64(10)); + builder.UnsafeAppend(ui64(20)); + builder.UnsafeAppend(ui64(30)); + std::shared_ptr<arrow::ArrayData> builderResult; + UNIT_ASSERT(builder.FinishInternal(&builderResult).ok()); + arrow::Datum d1(builderResult); + NUdf::TUnboxedValue val1 = HolderFactory.CreateArrowBlock(std::move(d1)); + + bool isScalar; + ui64 length; + auto chunks = Builder.GetArrowBlockChunks(val1, isScalar, length); + UNIT_ASSERT_VALUES_EQUAL(chunks, 1); + UNIT_ASSERT(!isScalar); + UNIT_ASSERT_VALUES_EQUAL(length, 3); + + ArrowArray arr1; + Builder.ExportArrowBlock(val1, 0, &arr1); + NUdf::TUnboxedValue val2 = Builder.ImportArrowBlock(&arr1, 1, isScalar, *atype); + const auto& d2 = TArrowBlock::From(val2).GetDatum(); + UNIT_ASSERT(d2.is_array()); + UNIT_ASSERT_VALUES_EQUAL(d2.array()->length, 3); + UNIT_ASSERT_VALUES_EQUAL(d2.array()->GetNullCount(), 0); + auto flat = d2.array()->GetValues<ui64>(1); + UNIT_ASSERT_VALUES_EQUAL(flat[0], 10); + UNIT_ASSERT_VALUES_EQUAL(flat[1], 20); + UNIT_ASSERT_VALUES_EQUAL(flat[2], 30); + } + + { + arrow::UInt64Builder builder1; + UNIT_ASSERT(builder1.Reserve(3).ok()); + builder1.UnsafeAppend(ui64(10)); + builder1.UnsafeAppend(ui64(20)); + builder1.UnsafeAppend(ui64(30)); + std::shared_ptr<arrow::Array> builder1Result; + UNIT_ASSERT(builder1.Finish(&builder1Result).ok()); + + arrow::UInt64Builder builder2; + UNIT_ASSERT(builder2.Reserve(2).ok()); + builder2.UnsafeAppend(ui64(40)); + builder2.UnsafeAppend(ui64(50)); + std::shared_ptr<arrow::Array> builder2Result; + UNIT_ASSERT(builder2.Finish(&builder2Result).ok()); + + auto chunked = arrow::ChunkedArray::Make({ builder1Result, builder2Result }).ValueOrDie(); + arrow::Datum d1(chunked); + NUdf::TUnboxedValue val1 = HolderFactory.CreateArrowBlock(std::move(d1)); + + bool isScalar; + ui64 length; + auto chunks = Builder.GetArrowBlockChunks(val1, isScalar, length); + UNIT_ASSERT_VALUES_EQUAL(chunks, 2); + UNIT_ASSERT(!isScalar); + UNIT_ASSERT_VALUES_EQUAL(length, 5); + + ArrowArray arrs[2]; + Builder.ExportArrowBlock(val1, 0, &arrs[0]); + Builder.ExportArrowBlock(val1, 1, &arrs[1]); + NUdf::TUnboxedValue val2 = Builder.ImportArrowBlock(arrs, 2, isScalar, *atype); + const auto& d2 = TArrowBlock::From(val2).GetDatum(); + UNIT_ASSERT(d2.is_arraylike() && !d2.is_array()); + UNIT_ASSERT_VALUES_EQUAL(d2.length(), 5); + UNIT_ASSERT_VALUES_EQUAL(d2.chunks().size(), 2); + + UNIT_ASSERT_VALUES_EQUAL(d2.chunks()[0]->data()->length, 3); + UNIT_ASSERT_VALUES_EQUAL(d2.chunks()[0]->data()->GetNullCount(), 0); + auto flat = d2.chunks()[0]->data()->GetValues<ui64>(1); + UNIT_ASSERT_VALUES_EQUAL(flat[0], 10); + UNIT_ASSERT_VALUES_EQUAL(flat[1], 20); + UNIT_ASSERT_VALUES_EQUAL(flat[2], 30); + + UNIT_ASSERT_VALUES_EQUAL(d2.chunks()[1]->data()->length, 2); + UNIT_ASSERT_VALUES_EQUAL(d2.chunks()[1]->data()->GetNullCount(), 0); + flat = d2.chunks()[1]->data()->GetValues<ui64>(1); + UNIT_ASSERT_VALUES_EQUAL(flat[0], 40); + UNIT_ASSERT_VALUES_EQUAL(flat[1], 50); + } + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TMiniKQLValueBuilderTest); + +} +} diff --git a/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..558b995f359 --- /dev/null +++ b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,52 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(minikql-computation-no_llvm) +target_compile_options(minikql-computation-no_llvm PRIVATE + -DMKQL_DISABLE_CODEGEN + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(minikql-computation-no_llvm PUBLIC + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub +) +target_link_libraries(minikql-computation-no_llvm PUBLIC + contrib-libs-cxxsupp + yutil + libs-apache-arrow + cpp-actors-util + library-cpp-enumbitset + library-cpp-packedtypes + library-cpp-random_provider + library-cpp-time_provider + library-yql-minikql + yql-minikql-arrow + parser-pg_wrapper-interface + yql-public-udf + library-yql-utils + cpp-threading-future +) +target_sources(minikql-computation-no_llvm PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_builder.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_impl.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_reader.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_transport.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_impl.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack_impl.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_custom_list.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_validate.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_value_builder.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/presort.cpp +) diff --git a/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..c2fbea09031 --- /dev/null +++ b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.linux-aarch64.txt @@ -0,0 +1,53 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(minikql-computation-no_llvm) +target_compile_options(minikql-computation-no_llvm PRIVATE + -DMKQL_DISABLE_CODEGEN + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(minikql-computation-no_llvm PUBLIC + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub +) +target_link_libraries(minikql-computation-no_llvm PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + libs-apache-arrow + cpp-actors-util + library-cpp-enumbitset + library-cpp-packedtypes + library-cpp-random_provider + library-cpp-time_provider + library-yql-minikql + yql-minikql-arrow + parser-pg_wrapper-interface + yql-public-udf + library-yql-utils + cpp-threading-future +) +target_sources(minikql-computation-no_llvm PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_builder.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_impl.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_reader.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_transport.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_impl.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack_impl.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_custom_list.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_validate.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_value_builder.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/presort.cpp +) diff --git a/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..c2fbea09031 --- /dev/null +++ b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.linux-x86_64.txt @@ -0,0 +1,53 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(minikql-computation-no_llvm) +target_compile_options(minikql-computation-no_llvm PRIVATE + -DMKQL_DISABLE_CODEGEN + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(minikql-computation-no_llvm PUBLIC + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub +) +target_link_libraries(minikql-computation-no_llvm PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + libs-apache-arrow + cpp-actors-util + library-cpp-enumbitset + library-cpp-packedtypes + library-cpp-random_provider + library-cpp-time_provider + library-yql-minikql + yql-minikql-arrow + parser-pg_wrapper-interface + yql-public-udf + library-yql-utils + cpp-threading-future +) +target_sources(minikql-computation-no_llvm PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_builder.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_impl.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_reader.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_transport.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_impl.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack_impl.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_custom_list.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_validate.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_value_builder.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/presort.cpp +) diff --git a/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.txt b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..558b995f359 --- /dev/null +++ b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.windows-x86_64.txt @@ -0,0 +1,52 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(minikql-computation-no_llvm) +target_compile_options(minikql-computation-no_llvm PRIVATE + -DMKQL_DISABLE_CODEGEN + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(minikql-computation-no_llvm PUBLIC + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub +) +target_link_libraries(minikql-computation-no_llvm PUBLIC + contrib-libs-cxxsupp + yutil + libs-apache-arrow + cpp-actors-util + library-cpp-enumbitset + library-cpp-packedtypes + library-cpp-random_provider + library-cpp-time_provider + library-yql-minikql + yql-minikql-arrow + parser-pg_wrapper-interface + yql-public-udf + library-yql-utils + cpp-threading-future +) +target_sources(minikql-computation-no_llvm PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_builder.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_impl.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_reader.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_transport.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_impl.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack_impl.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_custom_list.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_validate.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_value_builder.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/presort.cpp +) diff --git a/ydb/library/yql/minikql/computation/no_llvm/ya.make b/ydb/library/yql/minikql/computation/no_llvm/ya.make new file mode 100644 index 00000000000..7e86314b947 --- /dev/null +++ b/ydb/library/yql/minikql/computation/no_llvm/ya.make @@ -0,0 +1,9 @@ +LIBRARY() + +CXXFLAGS(-DMKQL_DISABLE_CODEGEN) + +ADDINCL(GLOBAL ydb/library/yql/minikql/codegen/llvm_stub) + +INCLUDE(../ya.make.inc) + +END() diff --git a/ydb/library/yql/minikql/computation/presort_ut.cpp b/ydb/library/yql/minikql/computation/presort_ut.cpp new file mode 100644 index 00000000000..0a6e537e4e8 --- /dev/null +++ b/ydb/library/yql/minikql/computation/presort_ut.cpp @@ -0,0 +1,649 @@ +#include "presort.h" + +#include <ydb/library/yql/minikql/mkql_alloc.h> +#include <ydb/library/yql/minikql/mkql_string_util.h> +#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/string/hex.h> + +using namespace std::literals::string_view_literals; + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + +#define TYPE_MAP(XX) \ + XX(bool, Bool) \ + XX(ui8, Uint8) \ + XX(ui16, Uint16) \ + XX(ui32, Uint32) \ + XX(ui64, Uint64) \ + XX(i8, Int8) \ + XX(i16, Int16) \ + XX(i32, Int32) \ + XX(i64, Int64) \ + XX(float, Float) \ + XX(double, Double) + +#define ALL_VALUES(xType, xName) \ + xType xName; + +struct TSimpleTypes { + TYPE_MAP(ALL_VALUES) +}; +#undef ALL_VALUES + +#define ADD_TYPE(xType, xName) \ + codec.AddType(NUdf::EDataSlot::xName, isOptional, isDesc); + +void AddTypes(TPresortCodec& codec, bool isOptional, bool isDesc) { + TYPE_MAP(ADD_TYPE) +} +#undef ADD_TYPE + +#define ENCODE(xType, xName) \ + encoder.Encode(NUdf::TUnboxedValuePod(values.xName)); + +TStringBuf Encode(NKikimr::NMiniKQL::TPresortEncoder& encoder, const TSimpleTypes& values) { + encoder.Start(); + TYPE_MAP(ENCODE) + return encoder.Finish(); +} +#undef ENCODE + +#define DECODE(xType, xName) \ + UNIT_ASSERT_EQUAL(decoder.Decode().Get<xType>(), values.xName); + +void Decode(TPresortDecoder& decoder, TStringBuf input, const TSimpleTypes& values) { + decoder.Start(input); + TYPE_MAP(DECODE) + decoder.Finish(); +} +#undef DECODE + +#undef TYPE_MAP + +struct TPresortTest { + TScopedAlloc Alloc; + TMemoryUsageInfo MemInfo; + + TPresortTest() + : Alloc(__LOCATION__) + , MemInfo("Memory") + {} + + template <typename T> + void ValidateEncoding(bool isDesc, T value, const TString& hex) { + TPresortEncoder encoder; + encoder.AddType(NUdf::TDataType<T>::Slot, false, isDesc); + + TPresortDecoder decoder; + decoder.AddType(NUdf::TDataType<T>::Slot, false, isDesc); + + encoder.Start(); + encoder.Encode(NUdf::TUnboxedValuePod(value)); + auto bytes = encoder.Finish(); + + UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex); + + decoder.Start(bytes); + auto decoded = decoder.Decode().Get<T>(); + decoder.Finish(); + + UNIT_ASSERT_EQUAL(decoded, value); + }; + + template <NUdf::EDataSlot Slot> + void ValidateEncoding(bool isDesc, TStringBuf value, const TString& hex) { + TPresortEncoder encoder; + encoder.AddType(Slot, false, isDesc); + + TPresortDecoder decoder; + decoder.AddType(Slot, false, isDesc); + + encoder.Start(); + encoder.Encode(NUdf::TUnboxedValue(MakeString(NUdf::TStringRef(value)))); + auto bytes = encoder.Finish(); + + UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex); + + decoder.Start(bytes); + auto uv = decoder.Decode(); + decoder.Finish(); + + auto stringRef = uv.AsStringRef(); + auto decoded = TStringBuf(stringRef.Data(), stringRef.Size()); + + UNIT_ASSERT_EQUAL(decoded, value); + } + + template <NUdf::EDataSlot Slot, typename T> + void ValidateEncoding(bool isDesc, const std::pair<T, ui16>& value, const TString& hex) { + TPresortEncoder encoder; + encoder.AddType(Slot, false, isDesc); + + TPresortDecoder decoder; + decoder.AddType(Slot, false, isDesc); + + NUdf::TUnboxedValuePod uv(value.first); + uv.SetTimezoneId(value.second); + + encoder.Start(); + encoder.Encode(uv); + auto bytes = encoder.Finish(); + + UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex); + + decoder.Start(bytes); + auto decoded = decoder.Decode(); + decoder.Finish(); + + UNIT_ASSERT_EQUAL(decoded.Get<T>(), value.first); + UNIT_ASSERT_EQUAL(decoded.GetTimezoneId(), value.second); + }; + + void ValidateEncoding(bool isDesc, NYql::NDecimal::TInt128 value, const TString& hex) { + TPresortEncoder encoder; + encoder.AddType(NUdf::EDataSlot::Decimal, false, isDesc); + + TPresortDecoder decoder; + decoder.AddType(NUdf::EDataSlot::Decimal, false, isDesc); + + encoder.Start(); + encoder.Encode(NUdf::TUnboxedValuePod(value)); + auto bytes = encoder.Finish(); + + UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex); + + decoder.Start(bytes); + auto decoded = decoder.Decode().GetInt128(); + decoder.Finish(); + + UNIT_ASSERT_EQUAL(decoded, value); + }; + + template <typename T> + void ValidateEncoding(const TVector<T>& values) { + for (auto& value : values) { + ValidateEncoding(false, std::get<0>(value), std::get<1>(value)); + ValidateEncoding(true, std::get<0>(value), std::get<2>(value)); + } + } + + template <NUdf::EDataSlot Slot, typename T> + void ValidateEncoding(const TVector<T>& values) { + for (auto& value : values) { + ValidateEncoding<Slot>(false, std::get<0>(value), std::get<1>(value)); + ValidateEncoding<Slot>(true, std::get<0>(value), std::get<2>(value)); + } + } +}; + +} + +Y_UNIT_TEST_SUITE(TPresortCodecTest) { + +Y_UNIT_TEST(SimpleTypes) { + TPresortTest test; + + TSimpleTypes values = {false, 1u, 2u, 3u, 4u, 5, 6, 7, 8, 9.f, 10.0}; + + auto validateSimpleTypes = [&] (bool isOptional, bool isDesc) { + TPresortEncoder encoder; + AddTypes(encoder, isOptional, isDesc); + + TPresortDecoder decoder; + AddTypes(decoder, isOptional, isDesc); + + auto bytes = Encode(encoder, values); + Decode(decoder, bytes, values); + }; + + validateSimpleTypes(false, false); + validateSimpleTypes(false, true); + validateSimpleTypes(true, false); + validateSimpleTypes(true, true); +} + + +Y_UNIT_TEST(Bool) { + const TVector<std::tuple<bool, TString, TString>> values = { + {false, "00", "FF"}, + {true, "01", "FE"} + }; + TPresortTest().ValidateEncoding(values); +} + +Y_UNIT_TEST(Int8) { + const TVector<std::tuple<i8, TString, TString>> values = { + {-0x80, "00", "FF"}, + {-1, "7F", "80"}, + {0, "80", "7F"}, + {1, "81", "7E"}, + {0x7F, "FF", "00"} + }; + TPresortTest().ValidateEncoding(values); +} + +Y_UNIT_TEST(Uint8) { + const TVector<std::tuple<ui8, TString, TString>> values = { + {0u, "00", "FF"}, + {0x80u, "80", "7F"}, + {0xFFu, "FF", "00"}, + }; + TPresortTest().ValidateEncoding(values); +} + +Y_UNIT_TEST(Int16) { + const TVector<std::tuple<i16, TString, TString>> values = { + {-0x8000, "0000", "FFFF"}, + {-1, "7FFF", "8000"}, + {0, "8000", "7FFF"}, + {1, "8001", "7FFE"}, + {0x7FFF, "FFFF", "0000"} + }; + TPresortTest().ValidateEncoding(values); +} + +Y_UNIT_TEST(Uint16) { + const TVector<std::tuple<ui16, TString, TString>> values = { + {0, "0000", "FFFF"}, + {0x8000u, "8000", "7FFF"}, + {0xFFFFu, "FFFF", "0000"}, + }; + TPresortTest().ValidateEncoding(values); +} + +Y_UNIT_TEST(Int32) { + const TVector<std::tuple<i32, TString, TString>> values = { + {-0x80000000, "00000000", "FFFFFFFF"}, + {-1, "7FFFFFFF", "80000000"}, + {0, "80000000", "7FFFFFFF"}, + {1, "80000001", "7FFFFFFE"}, + {0x7FFFFFFF, "FFFFFFFF", "00000000"} + }; + TPresortTest().ValidateEncoding(values); +} + +Y_UNIT_TEST(Uint32) { + const TVector<std::tuple<ui32, TString, TString>> values = { + {0u, "00000000", "FFFFFFFF"}, + {0x80000000u, "80000000", "7FFFFFFF"}, + {0xFFFFFFFFu, "FFFFFFFF", "00000000"}, + }; + TPresortTest().ValidateEncoding(values); +} + +Y_UNIT_TEST(Int64) { + const TVector<std::tuple<i64, TString, TString>> values = { + {-0x8000000000000000, "0000000000000000", "FFFFFFFFFFFFFFFF"}, + {-1, "7FFFFFFFFFFFFFFF", "8000000000000000"}, + {0, "8000000000000000", "7FFFFFFFFFFFFFFF"}, + {1, "8000000000000001", "7FFFFFFFFFFFFFFE"}, + {0x7FFFFFFFFFFFFFFF, "FFFFFFFFFFFFFFFF", "0000000000000000"} + }; + TPresortTest().ValidateEncoding(values); +} + +Y_UNIT_TEST(Uint64) { + const TVector<std::tuple<ui64, TString, TString>> values = { + {0u, "0000000000000000", "FFFFFFFFFFFFFFFF"}, + {0x8000000000000000u, "8000000000000000", "7FFFFFFFFFFFFFFF"}, + {0xFFFFFFFFFFFFFFFFu, "FFFFFFFFFFFFFFFF", "0000000000000000"} + }; + TPresortTest().ValidateEncoding(values); +} + +Y_UNIT_TEST(Float) { + using TLimits = std::numeric_limits<float>; + + const TVector<std::tuple<float, TString, TString>> values = { + {-TLimits::infinity(), "00", "FF"}, + {-TLimits::max(), "0100800000", "FEFF7FFFFF"}, + {-1.f, "01407FFFFF", "FEBF800000"}, + {-TLimits::min(), "017F7FFFFF", "FE80800000"}, + {-TLimits::min()/8.f, "017FEFFFFF", "FE80100000"}, + {0.f, "02", "FD"}, + {TLimits::min()/8.f, "0300100000", "FCFFEFFFFF"}, + {TLimits::min(), "0300800000", "FCFF7FFFFF"}, + {1.f, "033F800000", "FCC07FFFFF"}, + {TLimits::max(), "037F7FFFFF", "FC80800000"}, + {TLimits::infinity(), "04", "FB"}, + }; + TPresortTest().ValidateEncoding(values); +} + +Y_UNIT_TEST(Double) { + using TLimits = std::numeric_limits<double>; + + const TVector<std::tuple<double, TString, TString>> values = { + {-TLimits::infinity(), "00", "FF"}, + {-TLimits::max(), "010010000000000000", "FEFFEFFFFFFFFFFFFF"}, + {-1., "01400FFFFFFFFFFFFF", "FEBFF0000000000000"}, + {-TLimits::min(), "017FEFFFFFFFFFFFFF", "FE8010000000000000"}, + {-TLimits::min()/8., "017FFDFFFFFFFFFFFF", "FE8002000000000000"}, + {0., "02", "FD"}, + {TLimits::min()/8., "030002000000000000", "FCFFFDFFFFFFFFFFFF"}, + {TLimits::min(), "030010000000000000", "FCFFEFFFFFFFFFFFFF"}, + {1., "033FF0000000000000", "FCC00FFFFFFFFFFFFF"}, + {TLimits::max(), "037FEFFFFFFFFFFFFF", "FC8010000000000000"}, + {TLimits::infinity(), "04", "FB"}, + }; + TPresortTest().ValidateEncoding(values); +} + +Y_UNIT_TEST(String) { + const TVector<std::tuple<TStringBuf, TString, TString>> values = { + {TStringBuf(""), "00", "FF"}, + {"\x00"sv, "1F00000000000000000000000000000001", + "E0FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"}, + {"\x01", "1F01000000000000000000000000000001", + "E0FEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"}, + {"0", "1F30000000000000000000000000000001", + "E0CFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"}, + {"0123", "1F30313233000000000000000000000004", + "E0CFCECDCCFFFFFFFFFFFFFFFFFFFFFFFB"}, + {"0123456789abcde", "1F3031323334353637383961626364650F", + "E0CFCECDCCCBCAC9C8C7C69E9D9C9B9AF0"}, + {"a", "1F61000000000000000000000000000001", + "E09EFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"}, + {"a\x00"sv, "1F61000000000000000000000000000002", + "E09EFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD"}, + {"abc", "1F61626300000000000000000000000003", + "E09E9D9CFFFFFFFFFFFFFFFFFFFFFFFFFC"}, + {"b", "1F62000000000000000000000000000001", + "E09DFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"}, + }; + TPresortTest().ValidateEncoding<NUdf::EDataSlot::String>(values); +} + +Y_UNIT_TEST(Uuid) { + const TVector<std::tuple<TStringBuf, TString, TString>> values = { + {"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"sv, + "00000000000000000000000000000000", + "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"}, + {"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1"sv, + "00000000000000000000000000000001", + "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"}, + {"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", + "00000000000000000000000000000000"}, + }; + TPresortTest().ValidateEncoding<NUdf::EDataSlot::Uuid>(values); +} + +Y_UNIT_TEST(TzDate) { + const TVector<std::tuple<std::pair<ui16, ui16>, TString, TString>> values = { + {{0u, 0u}, "00000000", "FFFFFFFF"}, + {{0u, 1u}, "00000001", "FFFFFFFE"}, + {{1u, 0u}, "00010000", "FFFEFFFF"}, + {{NUdf::MAX_DATE, 0u}, "C2090000", "3DF6FFFF"}, + }; + TPresortTest().ValidateEncoding<NUdf::EDataSlot::TzDate>(values); +} + +Y_UNIT_TEST(TzDatetime) { + const TVector<std::tuple<std::pair<ui32, ui16>, TString, TString>> values = { + {{0u, 0u}, "000000000000", "FFFFFFFFFFFF"}, + {{0u, 1u}, "000000000001", "FFFFFFFFFFFE"}, + {{1u, 0u}, "000000010000", "FFFFFFFEFFFF"}, + {{NUdf::MAX_DATETIME, 0u}, "FFCEDD800000", "0031227FFFFF"}, + }; + TPresortTest().ValidateEncoding<NUdf::EDataSlot::TzDatetime>(values); +} + +Y_UNIT_TEST(TzTimestamp) { + const TVector<std::tuple<std::pair<ui64, ui16>, TString, TString>> values = { + {{0u, 0u}, "00000000000000000000", "FFFFFFFFFFFFFFFFFFFF"}, + {{0u, 1u}, "00000000000000000001", "FFFFFFFFFFFFFFFFFFFE"}, + {{1u, 0u}, "00000000000000010000", "FFFFFFFFFFFFFFFEFFFF"}, + {{NUdf::MAX_TIMESTAMP, 0u}, "000F3F52435260000000", "FFF0C0ADBCAD9FFFFFFF"}, + }; + TPresortTest().ValidateEncoding<NUdf::EDataSlot::TzTimestamp>(values); +} + +Y_UNIT_TEST(Decimal) { + const TVector<std::tuple<NYql::NDecimal::TInt128, TString, TString>> values = { + {-NYql::NDecimal::Nan(), + "00", + "FF"}, + {-NYql::NDecimal::Inf(), + "01", + "FE"}, + {NYql::NDecimal::TInt128(-1), + "7F", + "8101"}, + {NYql::NDecimal::TInt128(0), + "80", + "80"}, + {NYql::NDecimal::TInt128(1), + "8101", + "7F"}, + {NYql::NDecimal::Inf(), + "FE", + "01"}, + {NYql::NDecimal::Nan(), + "FF", + "00"}, + }; + TPresortTest().ValidateEncoding(values); +} + +Y_UNIT_TEST(GenericVoid) { + TScopedAlloc alloc(__LOCATION__); + TTypeEnvironment env(alloc); + auto type = env.GetVoid()->GetType(); + NUdf::TUnboxedValue value = NUdf::TUnboxedValuePod::Void(); + TGenericPresortEncoder encoder(type); + auto buf = encoder.Encode(value, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("")); +} + +Y_UNIT_TEST(GenericBool) { + TScopedAlloc alloc(__LOCATION__); + TTypeEnvironment env(alloc); + auto type = TDataType::Create(NUdf::TDataType<bool>::Id, env); + NUdf::TUnboxedValue value = NUdf::TUnboxedValuePod(true); + TGenericPresortEncoder encoder(type); + auto buf = encoder.Encode(value, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01")); + buf = encoder.Encode(value, true); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\xFE")); +} + +Y_UNIT_TEST(GenericNumber) { + TScopedAlloc alloc(__LOCATION__); + TTypeEnvironment env(alloc); + auto type = TDataType::Create(NUdf::TDataType<ui32>::Id, env); + NUdf::TUnboxedValue value = NUdf::TUnboxedValuePod(ui32(1234)); + TGenericPresortEncoder encoder(type); + auto buf = encoder.Encode(value, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x00\x00\x04\xD2"sv)); +} + +Y_UNIT_TEST(GenericString) { + TScopedAlloc alloc(__LOCATION__); + TTypeEnvironment env(alloc); + auto type = TDataType::Create(NUdf::TDataType<char*>::Id, env); + NUdf::TUnboxedValue value = MakeString("ALongStringExample"); + TGenericPresortEncoder encoder(type); + auto buf = encoder.Encode(value, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x1F" "ALongStringExam\x1Fple\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03"sv)); +} + +Y_UNIT_TEST(GenericOptional) { + TScopedAlloc alloc(__LOCATION__); + TTypeEnvironment env(alloc); + auto type = TOptionalType::Create(TDataType::Create(NUdf::TDataType<bool>::Id, env), env); + NUdf::TUnboxedValue value = NUdf::TUnboxedValuePod(true); + TGenericPresortEncoder encoder(type); + auto buf = encoder.Encode(value, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x01")); + value = {}; + buf = encoder.Encode(value, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x00"sv)); +} + +Y_UNIT_TEST(NestedOptional) { + TScopedAlloc alloc(__LOCATION__); + TTypeEnvironment env(alloc); + // Int32??? + auto type = + TOptionalType::Create(TOptionalType::Create(TOptionalType::Create(TDataType::Create(NUdf::TDataType<i32>::Id, env), env), env), env); + TGenericPresortEncoder encoder(type); + + NUdf::TUnboxedValue null = {}; + auto buf = encoder.Encode(null, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x00"sv)); + + auto justNull = null.MakeOptional(); + buf = encoder.Encode(justNull, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x00"sv)); + + auto justJustNull = justNull.MakeOptional(); + buf = encoder.Encode(justJustNull, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x01\x00"sv)); + + + auto zero = NUdf::TUnboxedValuePod(0).MakeOptional().MakeOptional().MakeOptional(); + buf = encoder.Encode(zero, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x01\x01\x80\x00\x00\x00"sv)); +} + + +Y_UNIT_TEST(GenericList) { + TScopedAlloc alloc(__LOCATION__); + TTypeEnvironment env(alloc); + auto type = TListType::Create(TDataType::Create(NUdf::TDataType<bool>::Id, env), env); + TMemoryUsageInfo memInfo("test"); + THolderFactory holderFactory(alloc.Ref(), memInfo); + auto value = holderFactory.GetEmptyContainer(); + TGenericPresortEncoder encoder(type); + auto buf = encoder.Encode(value, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x00"sv)); + NUdf::TUnboxedValue* items; + value = holderFactory.CreateDirectArrayHolder(1, items); + items[0] = NUdf::TUnboxedValuePod(true); + buf = encoder.Encode(value, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x01\x00"sv)); + value = holderFactory.CreateDirectArrayHolder(2, items); + items[0] = NUdf::TUnboxedValuePod(true); + items[1] = NUdf::TUnboxedValuePod(false); + buf = encoder.Encode(value, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x01\x01\x00\x00"sv)); +} + +Y_UNIT_TEST(GenericTuple) { + TScopedAlloc alloc(__LOCATION__); + TTypeEnvironment env(alloc); + TType* tupleTypes[2]; + tupleTypes[0] = TDataType::Create(NUdf::TDataType<bool>::Id, env); + tupleTypes[1] = TDataType::Create(NUdf::TDataType<ui32>::Id, env); + auto type = TTupleType::Create(2, tupleTypes, env); + TMemoryUsageInfo memInfo("test"); + THolderFactory holderFactory(alloc.Ref(), memInfo); + NUdf::TUnboxedValue* items; + auto value = holderFactory.CreateDirectArrayHolder(2, items); + items[0] = NUdf::TUnboxedValuePod(true); + items[1] = NUdf::TUnboxedValuePod(ui32(1234)); + TGenericPresortEncoder encoder(type); + auto buf = encoder.Encode(value, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x00\x00\x04\xD2"sv)); +} + +Y_UNIT_TEST(GenericStruct) { + TScopedAlloc alloc(__LOCATION__); + TTypeEnvironment env(alloc); + TStructMember structTypes[2]; + structTypes[0] = TStructMember("A", TDataType::Create(NUdf::TDataType<bool>::Id, env)); + structTypes[1] = TStructMember("B", TDataType::Create(NUdf::TDataType<ui32>::Id, env)); + auto type = TStructType::Create(2, structTypes, env); + TMemoryUsageInfo memInfo("test"); + THolderFactory holderFactory(alloc.Ref(), memInfo); + NUdf::TUnboxedValue* items; + auto value = holderFactory.CreateDirectArrayHolder(2, items); + items[0] = NUdf::TUnboxedValuePod(true); + items[1] = NUdf::TUnboxedValuePod(ui32(1234)); + TGenericPresortEncoder encoder(type); + auto buf = encoder.Encode(value, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x00\x00\x04\xD2"sv)); +} + +Y_UNIT_TEST(GenericTupleVariant) { + TScopedAlloc alloc(__LOCATION__); + TTypeEnvironment env(alloc); + TType* tupleTypes[2]; + tupleTypes[0] = TDataType::Create(NUdf::TDataType<bool>::Id, env); + tupleTypes[1] = TDataType::Create(NUdf::TDataType<ui32>::Id, env); + auto underlying = TTupleType::Create(2, tupleTypes, env); + auto type = TVariantType::Create(underlying, env); + TMemoryUsageInfo memInfo("test"); + THolderFactory holderFactory(alloc.Ref(), memInfo); + TGenericPresortEncoder encoder(type); + auto value = holderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(true), 0); + auto buf = encoder.Encode(value, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x00\x01"sv)); + value = holderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(ui32(1234)), 1); + buf = encoder.Encode(value, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x00\x00\x04\xD2"sv)); +} + +Y_UNIT_TEST(GenericStructVariant) { + TScopedAlloc alloc(__LOCATION__); + TTypeEnvironment env(alloc); + TStructMember structTypes[2]; + structTypes[0] = TStructMember("A", TDataType::Create(NUdf::TDataType<bool>::Id, env)); + structTypes[1] = TStructMember("B", TDataType::Create(NUdf::TDataType<ui32>::Id, env)); + auto underlying = TStructType::Create(2, structTypes, env); + auto type = TVariantType::Create(underlying, env); + TMemoryUsageInfo memInfo("test"); + THolderFactory holderFactory(alloc.Ref(), memInfo); + TGenericPresortEncoder encoder(type); + auto value = holderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(true), 0); + auto buf = encoder.Encode(value, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x00\x01"sv)); + value = holderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(ui32(1234)), 1); + buf = encoder.Encode(value, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x00\x00\x04\xD2"sv)); +} + +Y_UNIT_TEST(GenericDict) { + TScopedAlloc alloc(__LOCATION__); + TTypeEnvironment env(alloc); + auto type = TDictType::Create(TDataType::Create(NUdf::TDataType<ui32>::Id, env), + TDataType::Create(NUdf::TDataType<bool>::Id, env), env); + TKeyTypes keyTypes; + bool isTuple; + bool encoded; + bool useIHash; + GetDictionaryKeyTypes(type->GetKeyType(), keyTypes, isTuple, encoded, useIHash); + UNIT_ASSERT(!isTuple); + UNIT_ASSERT(!encoded); + UNIT_ASSERT(!useIHash); + TMemoryUsageInfo memInfo("test"); + THolderFactory holderFactory(alloc.Ref(), memInfo); + auto value = holderFactory.GetEmptyContainer(); + TGenericPresortEncoder encoder(type); + auto buf = encoder.Encode(value, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x00"sv)); + value = holderFactory.CreateDirectHashedDictHolder([](TValuesDictHashMap& map) { + map.emplace(NUdf::TUnboxedValuePod(ui32(1234)), NUdf::TUnboxedValuePod(true)); + }, keyTypes, false, true, nullptr, nullptr, nullptr); + buf = encoder.Encode(value, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x00\x00\x04\xD2\x01\x00"sv)); + value = holderFactory.CreateDirectHashedDictHolder([](TValuesDictHashMap& map) { + map.emplace(NUdf::TUnboxedValuePod(ui32(5678)), NUdf::TUnboxedValuePod(false)); + map.emplace(NUdf::TUnboxedValuePod(ui32(1234)), NUdf::TUnboxedValuePod(true)); + }, keyTypes, false, true, nullptr, nullptr, nullptr); + buf = encoder.Encode(value, false); + UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x00\x00\x04\xD2\x01\x01\x00\x00\x16\x2E\x00\x00"sv)); +} + +} + +} // NMiniKQL +} // NKikimr diff --git a/ydb/library/yql/minikql/computation/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/computation/ut/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..01dba3f99d8 --- /dev/null +++ b/ydb/library/yql/minikql/computation/ut/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,90 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-computation-ut) +target_compile_options(ydb-library-yql-minikql-computation-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-computation-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/llvm +) +target_link_libraries(ydb-library-yql-minikql-computation-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + minikql-computation-llvm + libs-apache-arrow + cpp-threading-local_executor + minikql-comp_nodes-llvm + yql-parser-pg_wrapper + udf-service-exception_policy + yql-dq-proto +) +target_link_options(ydb-library-yql-minikql-computation-ut PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-library-yql-minikql-computation-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_list_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_validate_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_value_builder_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/presort_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-computation-ut + PROPERTY + SPLIT_FACTOR + 10 +) +add_yunittest( + NAME + ydb-library-yql-minikql-computation-ut + TEST_TARGET + ydb-library-yql-minikql-computation-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-computation-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-computation-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-computation-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-minikql-computation-ut + system_allocator +) +vcs_info(ydb-library-yql-minikql-computation-ut) diff --git a/ydb/library/yql/minikql/computation/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/computation/ut/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..67eac9242ad --- /dev/null +++ b/ydb/library/yql/minikql/computation/ut/CMakeLists.linux-aarch64.txt @@ -0,0 +1,93 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-computation-ut) +target_compile_options(ydb-library-yql-minikql-computation-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-computation-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/llvm +) +target_link_libraries(ydb-library-yql-minikql-computation-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + minikql-computation-llvm + libs-apache-arrow + cpp-threading-local_executor + minikql-comp_nodes-llvm + yql-parser-pg_wrapper + udf-service-exception_policy + yql-dq-proto +) +target_link_options(ydb-library-yql-minikql-computation-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-minikql-computation-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_list_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_validate_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_value_builder_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/presort_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-computation-ut + PROPERTY + SPLIT_FACTOR + 10 +) +add_yunittest( + NAME + ydb-library-yql-minikql-computation-ut + TEST_TARGET + ydb-library-yql-minikql-computation-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-computation-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-computation-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-computation-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-minikql-computation-ut + cpp-malloc-jemalloc +) +vcs_info(ydb-library-yql-minikql-computation-ut) diff --git a/ydb/library/yql/minikql/computation/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/computation/ut/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..436ad6ceac5 --- /dev/null +++ b/ydb/library/yql/minikql/computation/ut/CMakeLists.linux-x86_64.txt @@ -0,0 +1,95 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-computation-ut) +target_compile_options(ydb-library-yql-minikql-computation-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-computation-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/llvm +) +target_link_libraries(ydb-library-yql-minikql-computation-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + minikql-computation-llvm + libs-apache-arrow + cpp-threading-local_executor + minikql-comp_nodes-llvm + yql-parser-pg_wrapper + udf-service-exception_policy + yql-dq-proto +) +target_link_options(ydb-library-yql-minikql-computation-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-minikql-computation-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_list_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_validate_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_value_builder_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/presort_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-computation-ut + PROPERTY + SPLIT_FACTOR + 10 +) +add_yunittest( + NAME + ydb-library-yql-minikql-computation-ut + TEST_TARGET + ydb-library-yql-minikql-computation-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-computation-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-computation-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-computation-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-minikql-computation-ut + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(ydb-library-yql-minikql-computation-ut) diff --git a/ydb/library/yql/minikql/computation/ut/CMakeLists.txt b/ydb/library/yql/minikql/computation/ut/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/minikql/computation/ut/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/minikql/computation/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/computation/ut/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..4244aa874b3 --- /dev/null +++ b/ydb/library/yql/minikql/computation/ut/CMakeLists.windows-x86_64.txt @@ -0,0 +1,83 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-computation-ut) +target_compile_options(ydb-library-yql-minikql-computation-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-computation-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/llvm +) +target_link_libraries(ydb-library-yql-minikql-computation-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + minikql-computation-llvm + libs-apache-arrow + cpp-threading-local_executor + minikql-comp_nodes-llvm + yql-parser-pg_wrapper + udf-service-exception_policy + yql-dq-proto +) +target_sources(ydb-library-yql-minikql-computation-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_list_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_validate_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_value_builder_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/presort_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-computation-ut + PROPERTY + SPLIT_FACTOR + 10 +) +add_yunittest( + NAME + ydb-library-yql-minikql-computation-ut + TEST_TARGET + ydb-library-yql-minikql-computation-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-computation-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-computation-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-computation-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-minikql-computation-ut + system_allocator +) +vcs_info(ydb-library-yql-minikql-computation-ut) diff --git a/ydb/library/yql/minikql/computation/ut/ya.make b/ydb/library/yql/minikql/computation/ut/ya.make new file mode 100644 index 00000000000..876dc95ff82 --- /dev/null +++ b/ydb/library/yql/minikql/computation/ut/ya.make @@ -0,0 +1,38 @@ +UNITTEST_FOR(ydb/library/yql/minikql/computation/llvm) + +FORK_SUBTESTS() + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + TIMEOUT(1800) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +SRCDIR(ydb/library/yql/minikql/computation) + +SRCS( + mkql_computation_node_pack_ut.cpp + mkql_computation_node_list_ut.cpp + mkql_computation_node_dict_ut.cpp + mkql_computation_node_graph_saveload_ut.cpp + mkql_computation_pattern_cache_ut.cpp + mkql_validate_ut.cpp + mkql_value_builder_ut.cpp + presort_ut.cpp +) + +PEERDIR( + contrib/libs/apache/arrow + library/cpp/threading/local_executor + ydb/library/yql/minikql/comp_nodes/llvm + ydb/library/yql/parser/pg_wrapper + ydb/library/yql/public/udf/service/exception_policy + ydb/library/yql/dq/proto +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/library/yql/minikql/computation/ya.make b/ydb/library/yql/minikql/computation/ya.make new file mode 100644 index 00000000000..424ecfb86d0 --- /dev/null +++ b/ydb/library/yql/minikql/computation/ya.make @@ -0,0 +1,8 @@ +RECURSE( + llvm + no_llvm +) + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/library/yql/minikql/dom/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/dom/CMakeLists.darwin-x86_64.txt index 4d9af8a9e92..5ab46c00cdf 100644 --- a/ydb/library/yql/minikql/dom/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/minikql/dom/CMakeLists.darwin-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) add_library(yql-minikql-dom) target_compile_options(yql-minikql-dom PRIVATE diff --git a/ydb/library/yql/minikql/dom/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/dom/CMakeLists.linux-aarch64.txt index 5d15504310f..c6d55bf198d 100644 --- a/ydb/library/yql/minikql/dom/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/minikql/dom/CMakeLists.linux-aarch64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) add_library(yql-minikql-dom) target_compile_options(yql-minikql-dom PRIVATE diff --git a/ydb/library/yql/minikql/dom/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/dom/CMakeLists.linux-x86_64.txt index 5d15504310f..c6d55bf198d 100644 --- a/ydb/library/yql/minikql/dom/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/minikql/dom/CMakeLists.linux-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) add_library(yql-minikql-dom) target_compile_options(yql-minikql-dom PRIVATE diff --git a/ydb/library/yql/minikql/dom/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/dom/ut/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..2fd86a151a5 --- /dev/null +++ b/ydb/library/yql/minikql/dom/ut/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,74 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-dom-ut) +target_compile_options(ydb-library-yql-minikql-dom-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-dom-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/dom +) +target_link_libraries(ydb-library-yql-minikql-dom-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-minikql-dom + minikql-computation-llvm + udf-service-exception_policy + yql-sql-pg_dummy +) +target_link_options(ydb-library-yql-minikql-dom-ut PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-library-yql-minikql-dom-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/dom/ut/yson_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/dom/ut/json_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-dom-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-minikql-dom-ut + TEST_TARGET + ydb-library-yql-minikql-dom-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-dom-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-dom-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-minikql-dom-ut + system_allocator +) +vcs_info(ydb-library-yql-minikql-dom-ut) diff --git a/ydb/library/yql/minikql/dom/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/dom/ut/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..3fefeffccc1 --- /dev/null +++ b/ydb/library/yql/minikql/dom/ut/CMakeLists.linux-aarch64.txt @@ -0,0 +1,77 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-dom-ut) +target_compile_options(ydb-library-yql-minikql-dom-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-dom-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/dom +) +target_link_libraries(ydb-library-yql-minikql-dom-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + yql-minikql-dom + minikql-computation-llvm + udf-service-exception_policy + yql-sql-pg_dummy +) +target_link_options(ydb-library-yql-minikql-dom-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-minikql-dom-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/dom/ut/yson_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/dom/ut/json_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-dom-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-minikql-dom-ut + TEST_TARGET + ydb-library-yql-minikql-dom-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-dom-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-dom-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-minikql-dom-ut + cpp-malloc-jemalloc +) +vcs_info(ydb-library-yql-minikql-dom-ut) diff --git a/ydb/library/yql/minikql/dom/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/dom/ut/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..739b1984df5 --- /dev/null +++ b/ydb/library/yql/minikql/dom/ut/CMakeLists.linux-x86_64.txt @@ -0,0 +1,79 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-dom-ut) +target_compile_options(ydb-library-yql-minikql-dom-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-dom-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/dom +) +target_link_libraries(ydb-library-yql-minikql-dom-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-minikql-dom + minikql-computation-llvm + udf-service-exception_policy + yql-sql-pg_dummy +) +target_link_options(ydb-library-yql-minikql-dom-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-minikql-dom-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/dom/ut/yson_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/dom/ut/json_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-dom-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-minikql-dom-ut + TEST_TARGET + ydb-library-yql-minikql-dom-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-dom-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-dom-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-minikql-dom-ut + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(ydb-library-yql-minikql-dom-ut) diff --git a/ydb/library/yql/minikql/dom/ut/CMakeLists.txt b/ydb/library/yql/minikql/dom/ut/CMakeLists.txt new file mode 100644 index 00000000000..606ff46b4be --- /dev/null +++ b/ydb/library/yql/minikql/dom/ut/CMakeLists.txt @@ -0,0 +1,15 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/minikql/dom/ut/ya.make b/ydb/library/yql/minikql/dom/ut/ya.make index 2386e8e0696..b6e80f6c7b0 100644 --- a/ydb/library/yql/minikql/dom/ut/ya.make +++ b/ydb/library/yql/minikql/dom/ut/ya.make @@ -1,5 +1,5 @@ IF (NOT WINDOWS) - UNITTEST_FOR(yql/library/dom) + UNITTEST_FOR(ydb/library/yql/minikql/dom) SRCS( yson_ut.cpp @@ -14,6 +14,7 @@ IF (NOT WINDOWS) PEERDIR( ydb/library/yql/minikql/computation/llvm ydb/library/yql/public/udf/service/exception_policy + ydb/library/yql/sql/pg_dummy ) YQL_LAST_ABI_VERSION() diff --git a/ydb/library/yql/minikql/invoke_builtins/CMakeLists.txt b/ydb/library/yql/minikql/invoke_builtins/CMakeLists.txt index e519707fa41..4716de1ef37 100644 --- a/ydb/library/yql/minikql/invoke_builtins/CMakeLists.txt +++ b/ydb/library/yql/minikql/invoke_builtins/CMakeLists.txt @@ -7,3 +7,5 @@ add_subdirectory(llvm) +add_subdirectory(no_llvm) +add_subdirectory(ut) diff --git a/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_ut.cpp b/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_ut.cpp new file mode 100644 index 00000000000..9420223b74e --- /dev/null +++ b/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_ut.cpp @@ -0,0 +1,51 @@ +#include "mkql_builtins.h" + +#include <ydb/library/yql/public/udf/udf_value.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <array> + +namespace NKikimr { +namespace NMiniKQL { + +static TFunctionParamMetadata AddUi32Metadata[] = { + { NUdf::TDataType<ui32>::Id, 0 }, // result + { NUdf::TDataType<ui32>::Id, 0 }, // first arg + { NUdf::TDataType<ui32>::Id, 0 }, // second arg + { 0, 0 } +}; + +static NUdf::TUnboxedValuePod AddUi32(const NUdf::TUnboxedValuePod* args) +{ + const ui32 first = args[0].Get<ui32>(); + const ui32 second = args[1].Get<ui32>(); + return NUdf::TUnboxedValuePod(first + second); +} + +Y_UNIT_TEST_SUITE(TFunctionRegistryTest) { + Y_UNIT_TEST(TestRegistration) { + const auto functionRegistry = CreateBuiltinRegistry(); + functionRegistry->Register("MyAdd", TFunctionDescriptor(AddUi32Metadata, &AddUi32)); + + const std::array<TArgType, 3U> argTypes ={{{ NUdf::TDataType<ui32>::Id, false }, { NUdf::TDataType<ui32>::Id, false }, { NUdf::TDataType<ui32>::Id, false }}}; + auto op = functionRegistry->GetBuiltin("MyAdd", argTypes.data(), argTypes.size()); + UNIT_ASSERT_EQUAL(op.Function, &AddUi32); + UNIT_ASSERT_EQUAL(op.ResultAndArgs[0].SchemeType, NUdf::TDataType<ui32>::Id); + UNIT_ASSERT_EQUAL(op.ResultAndArgs[0].Flags, 0); + UNIT_ASSERT_EQUAL(op.ResultAndArgs[1].SchemeType, NUdf::TDataType<ui32>::Id); + UNIT_ASSERT_EQUAL(op.ResultAndArgs[1].Flags, 0); + UNIT_ASSERT_EQUAL(op.ResultAndArgs[2].SchemeType, NUdf::TDataType<ui32>::Id); + UNIT_ASSERT_EQUAL(op.ResultAndArgs[2].Flags, 0); + UNIT_ASSERT_EQUAL(op.ResultAndArgs[3].SchemeType, 0); + + const NUdf::TUnboxedValuePod args[2] = {NUdf::TUnboxedValuePod(ui32(2)), NUdf::TUnboxedValuePod(ui32(3))}; + + auto result = op.Function(&args[0]); + UNIT_ASSERT_EQUAL(result.Get<ui32>(), 5); + } +} + +} // namespace NMiniKQL + +} // namespace NKikimr diff --git a/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..cd35db266b6 --- /dev/null +++ b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,69 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(minikql-invoke_builtins-no_llvm) +target_compile_options(minikql-invoke_builtins-no_llvm PRIVATE + -DMKQL_DISABLE_CODEGEN + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(minikql-invoke_builtins-no_llvm PUBLIC + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub +) +target_link_libraries(minikql-invoke_builtins-no_llvm PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-pop_count + ydb-library-binary_json + library-yql-minikql + yql-minikql-arrow + yql-public-udf + libs-apache-arrow + minikql-computation-no_llvm +) +target_sources(minikql-invoke_builtins-no_llvm PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_abs.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_add.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitand.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitnot.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitor.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitxor.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_byteat.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_codegen.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_concat.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_convert.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_countbits.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_decimal.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_dec.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_div.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_find.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_inc.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_invprestr.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_max.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_min.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_minus.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_mod.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_mul.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_nanvl.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_plus.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_rotleft.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_rotright.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_shiftleft.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_shiftright.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_string_kernels.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_sub.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_substring.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_with.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_equals.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_not_equals.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_less.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_less_or_equal.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_greater.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_greater_or_equal.cpp +) diff --git a/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..c71c6c41366 --- /dev/null +++ b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.linux-aarch64.txt @@ -0,0 +1,70 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(minikql-invoke_builtins-no_llvm) +target_compile_options(minikql-invoke_builtins-no_llvm PRIVATE + -DMKQL_DISABLE_CODEGEN + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(minikql-invoke_builtins-no_llvm PUBLIC + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub +) +target_link_libraries(minikql-invoke_builtins-no_llvm PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-pop_count + ydb-library-binary_json + library-yql-minikql + yql-minikql-arrow + yql-public-udf + libs-apache-arrow + minikql-computation-no_llvm +) +target_sources(minikql-invoke_builtins-no_llvm PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_abs.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_add.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitand.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitnot.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitor.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitxor.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_byteat.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_codegen.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_concat.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_convert.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_countbits.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_decimal.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_dec.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_div.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_find.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_inc.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_invprestr.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_max.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_min.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_minus.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_mod.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_mul.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_nanvl.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_plus.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_rotleft.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_rotright.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_shiftleft.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_shiftright.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_string_kernels.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_sub.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_substring.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_with.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_equals.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_not_equals.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_less.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_less_or_equal.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_greater.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_greater_or_equal.cpp +) diff --git a/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..c71c6c41366 --- /dev/null +++ b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.linux-x86_64.txt @@ -0,0 +1,70 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(minikql-invoke_builtins-no_llvm) +target_compile_options(minikql-invoke_builtins-no_llvm PRIVATE + -DMKQL_DISABLE_CODEGEN + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(minikql-invoke_builtins-no_llvm PUBLIC + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub +) +target_link_libraries(minikql-invoke_builtins-no_llvm PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-pop_count + ydb-library-binary_json + library-yql-minikql + yql-minikql-arrow + yql-public-udf + libs-apache-arrow + minikql-computation-no_llvm +) +target_sources(minikql-invoke_builtins-no_llvm PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_abs.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_add.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitand.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitnot.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitor.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitxor.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_byteat.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_codegen.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_concat.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_convert.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_countbits.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_decimal.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_dec.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_div.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_find.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_inc.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_invprestr.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_max.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_min.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_minus.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_mod.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_mul.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_nanvl.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_plus.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_rotleft.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_rotright.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_shiftleft.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_shiftright.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_string_kernels.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_sub.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_substring.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_with.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_equals.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_not_equals.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_less.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_less_or_equal.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_greater.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_greater_or_equal.cpp +) diff --git a/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.txt b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..cd35db266b6 --- /dev/null +++ b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.windows-x86_64.txt @@ -0,0 +1,69 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(minikql-invoke_builtins-no_llvm) +target_compile_options(minikql-invoke_builtins-no_llvm PRIVATE + -DMKQL_DISABLE_CODEGEN + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(minikql-invoke_builtins-no_llvm PUBLIC + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub +) +target_link_libraries(minikql-invoke_builtins-no_llvm PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-pop_count + ydb-library-binary_json + library-yql-minikql + yql-minikql-arrow + yql-public-udf + libs-apache-arrow + minikql-computation-no_llvm +) +target_sources(minikql-invoke_builtins-no_llvm PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_abs.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_add.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitand.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitnot.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitor.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitxor.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_byteat.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_codegen.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_concat.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_convert.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_countbits.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_decimal.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_dec.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_div.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_find.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_inc.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_invprestr.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_max.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_min.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_minus.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_mod.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_mul.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_nanvl.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_plus.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_rotleft.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_rotright.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_shiftleft.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_shiftright.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_string_kernels.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_sub.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_substring.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_with.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_equals.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_not_equals.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_less.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_less_or_equal.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_greater.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_greater_or_equal.cpp +) diff --git a/ydb/library/yql/minikql/invoke_builtins/no_llvm/ya.make b/ydb/library/yql/minikql/invoke_builtins/no_llvm/ya.make new file mode 100644 index 00000000000..0de0a0d6fd7 --- /dev/null +++ b/ydb/library/yql/minikql/invoke_builtins/no_llvm/ya.make @@ -0,0 +1,12 @@ +LIBRARY() + +CXXFLAGS(-DMKQL_DISABLE_CODEGEN) + +ADDINCL(GLOBAL ydb/library/yql/minikql/codegen/llvm_stub) + +INCLUDE(../ya.make.inc) + +PEERDIR(ydb/library/yql/minikql/computation/no_llvm) + +END() + diff --git a/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..29bde95f2eb --- /dev/null +++ b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,80 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-invoke_builtins-ut) +target_compile_options(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/llvm +) +target_link_libraries(ydb-library-yql-minikql-invoke_builtins-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + minikql-invoke_builtins-llvm + yql-public-udf + udf-service-exception_policy + yql-sql-pg_dummy +) +target_link_options(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-invoke_builtins-ut + PROPERTY + SPLIT_FACTOR + 10 +) +add_yunittest( + NAME + ydb-library-yql-minikql-invoke_builtins-ut + TEST_TARGET + ydb-library-yql-minikql-invoke_builtins-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-invoke_builtins-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-invoke_builtins-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-invoke_builtins-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-minikql-invoke_builtins-ut + system_allocator +) +vcs_info(ydb-library-yql-minikql-invoke_builtins-ut) diff --git a/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..6be7f1567aa --- /dev/null +++ b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.linux-aarch64.txt @@ -0,0 +1,83 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-invoke_builtins-ut) +target_compile_options(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/llvm +) +target_link_libraries(ydb-library-yql-minikql-invoke_builtins-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + minikql-invoke_builtins-llvm + yql-public-udf + udf-service-exception_policy + yql-sql-pg_dummy +) +target_link_options(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-invoke_builtins-ut + PROPERTY + SPLIT_FACTOR + 10 +) +add_yunittest( + NAME + ydb-library-yql-minikql-invoke_builtins-ut + TEST_TARGET + ydb-library-yql-minikql-invoke_builtins-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-invoke_builtins-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-invoke_builtins-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-invoke_builtins-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-minikql-invoke_builtins-ut + cpp-malloc-jemalloc +) +vcs_info(ydb-library-yql-minikql-invoke_builtins-ut) diff --git a/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..74923c7ad7d --- /dev/null +++ b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.linux-x86_64.txt @@ -0,0 +1,85 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-invoke_builtins-ut) +target_compile_options(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/llvm +) +target_link_libraries(ydb-library-yql-minikql-invoke_builtins-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + minikql-invoke_builtins-llvm + yql-public-udf + udf-service-exception_policy + yql-sql-pg_dummy +) +target_link_options(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-invoke_builtins-ut + PROPERTY + SPLIT_FACTOR + 10 +) +add_yunittest( + NAME + ydb-library-yql-minikql-invoke_builtins-ut + TEST_TARGET + ydb-library-yql-minikql-invoke_builtins-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-invoke_builtins-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-invoke_builtins-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-invoke_builtins-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-minikql-invoke_builtins-ut + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(ydb-library-yql-minikql-invoke_builtins-ut) diff --git a/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.txt b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..a3ad9ceabdc --- /dev/null +++ b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.windows-x86_64.txt @@ -0,0 +1,73 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-invoke_builtins-ut) +target_compile_options(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/llvm +) +target_link_libraries(ydb-library-yql-minikql-invoke_builtins-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + minikql-invoke_builtins-llvm + yql-public-udf + udf-service-exception_policy + yql-sql-pg_dummy +) +target_sources(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-invoke_builtins-ut + PROPERTY + SPLIT_FACTOR + 10 +) +add_yunittest( + NAME + ydb-library-yql-minikql-invoke_builtins-ut + TEST_TARGET + ydb-library-yql-minikql-invoke_builtins-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-invoke_builtins-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-invoke_builtins-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-invoke_builtins-ut + PROPERTY + TIMEOUT + 600 +) +target_allocator(ydb-library-yql-minikql-invoke_builtins-ut + system_allocator +) +vcs_info(ydb-library-yql-minikql-invoke_builtins-ut) diff --git a/ydb/library/yql/minikql/invoke_builtins/ut/ya.make b/ydb/library/yql/minikql/invoke_builtins/ut/ya.make new file mode 100644 index 00000000000..eb88ec513d5 --- /dev/null +++ b/ydb/library/yql/minikql/invoke_builtins/ut/ya.make @@ -0,0 +1,28 @@ +UNITTEST_FOR(ydb/library/yql/minikql/invoke_builtins/llvm) + +FORK_SUBTESTS() + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + TIMEOUT(1800) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +PEERDIR( + ydb/library/yql/public/udf + ydb/library/yql/public/udf/service/exception_policy + ydb/library/yql/sql/pg_dummy +) + +YQL_LAST_ABI_VERSION() + +SRCDIR(ydb/library/yql/minikql/invoke_builtins) + +SRCS( + mkql_builtins_ut.cpp +) + +END() diff --git a/ydb/library/yql/minikql/invoke_builtins/ya.make b/ydb/library/yql/minikql/invoke_builtins/ya.make new file mode 100644 index 00000000000..424ecfb86d0 --- /dev/null +++ b/ydb/library/yql/minikql/invoke_builtins/ya.make @@ -0,0 +1,8 @@ +RECURSE( + llvm + no_llvm +) + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/library/yql/minikql/jsonpath/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/jsonpath/CMakeLists.darwin-x86_64.txt index bb96e932dc2..5224e6130f6 100644 --- a/ydb/library/yql/minikql/jsonpath/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/minikql/jsonpath/CMakeLists.darwin-x86_64.txt @@ -7,6 +7,7 @@ add_subdirectory(benchmark) +add_subdirectory(ut) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/minikql/jsonpath/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/jsonpath/CMakeLists.linux-aarch64.txt index 09049a15249..a07d8bcad0a 100644 --- a/ydb/library/yql/minikql/jsonpath/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/minikql/jsonpath/CMakeLists.linux-aarch64.txt @@ -7,6 +7,7 @@ add_subdirectory(benchmark) +add_subdirectory(ut) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/minikql/jsonpath/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/jsonpath/CMakeLists.linux-x86_64.txt index 033eee4ae6c..64ac719f44e 100644 --- a/ydb/library/yql/minikql/jsonpath/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/minikql/jsonpath/CMakeLists.linux-x86_64.txt @@ -7,6 +7,7 @@ add_subdirectory(benchmark) +add_subdirectory(ut) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/minikql/jsonpath/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/jsonpath/CMakeLists.windows-x86_64.txt index bb96e932dc2..5224e6130f6 100644 --- a/ydb/library/yql/minikql/jsonpath/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/minikql/jsonpath/CMakeLists.windows-x86_64.txt @@ -7,6 +7,7 @@ add_subdirectory(benchmark) +add_subdirectory(ut) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..d130b466d35 --- /dev/null +++ b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,84 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-jsonpath-ut) +target_compile_options(ydb-library-yql-minikql-jsonpath-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-jsonpath-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath +) +target_link_libraries(ydb-library-yql-minikql-jsonpath-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-minikql-jsonpath + library-cpp-json + ydb-library-binary_json + library-yql-minikql + minikql-computation-llvm + yql-minikql-dom + minikql-invoke_builtins-llvm + udf-service-exception_policy + core-issue-protos + yql-sql-pg_dummy +) +target_link_options(ydb-library-yql-minikql-jsonpath-ut PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-library-yql-minikql-jsonpath-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/common_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/examples_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/lax_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/strict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/test_base.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/lib_id_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-jsonpath-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-minikql-jsonpath-ut + TEST_TARGET + ydb-library-yql-minikql-jsonpath-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-jsonpath-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-jsonpath-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-minikql-jsonpath-ut + system_allocator +) +vcs_info(ydb-library-yql-minikql-jsonpath-ut) diff --git a/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..da30de49141 --- /dev/null +++ b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.linux-aarch64.txt @@ -0,0 +1,87 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-jsonpath-ut) +target_compile_options(ydb-library-yql-minikql-jsonpath-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-jsonpath-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath +) +target_link_libraries(ydb-library-yql-minikql-jsonpath-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + yql-minikql-jsonpath + library-cpp-json + ydb-library-binary_json + library-yql-minikql + minikql-computation-llvm + yql-minikql-dom + minikql-invoke_builtins-llvm + udf-service-exception_policy + core-issue-protos + yql-sql-pg_dummy +) +target_link_options(ydb-library-yql-minikql-jsonpath-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-minikql-jsonpath-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/common_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/examples_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/lax_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/strict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/test_base.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/lib_id_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-jsonpath-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-minikql-jsonpath-ut + TEST_TARGET + ydb-library-yql-minikql-jsonpath-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-jsonpath-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-jsonpath-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-minikql-jsonpath-ut + cpp-malloc-jemalloc +) +vcs_info(ydb-library-yql-minikql-jsonpath-ut) diff --git a/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..a9532bd37b2 --- /dev/null +++ b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.linux-x86_64.txt @@ -0,0 +1,89 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-jsonpath-ut) +target_compile_options(ydb-library-yql-minikql-jsonpath-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-jsonpath-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath +) +target_link_libraries(ydb-library-yql-minikql-jsonpath-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-minikql-jsonpath + library-cpp-json + ydb-library-binary_json + library-yql-minikql + minikql-computation-llvm + yql-minikql-dom + minikql-invoke_builtins-llvm + udf-service-exception_policy + core-issue-protos + yql-sql-pg_dummy +) +target_link_options(ydb-library-yql-minikql-jsonpath-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-minikql-jsonpath-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/common_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/examples_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/lax_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/strict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/test_base.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/lib_id_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-jsonpath-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-minikql-jsonpath-ut + TEST_TARGET + ydb-library-yql-minikql-jsonpath-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-jsonpath-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-jsonpath-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-minikql-jsonpath-ut + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(ydb-library-yql-minikql-jsonpath-ut) diff --git a/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.txt b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..4c482b6b1df --- /dev/null +++ b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.windows-x86_64.txt @@ -0,0 +1,77 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-minikql-jsonpath-ut) +target_compile_options(ydb-library-yql-minikql-jsonpath-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_include_directories(ydb-library-yql-minikql-jsonpath-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath +) +target_link_libraries(ydb-library-yql-minikql-jsonpath-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-minikql-jsonpath + library-cpp-json + ydb-library-binary_json + library-yql-minikql + minikql-computation-llvm + yql-minikql-dom + minikql-invoke_builtins-llvm + udf-service-exception_policy + core-issue-protos + yql-sql-pg_dummy +) +target_sources(ydb-library-yql-minikql-jsonpath-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/common_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/examples_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/lax_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/strict_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/test_base.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/lib_id_ut.cpp +) +set_property( + TARGET + ydb-library-yql-minikql-jsonpath-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-minikql-jsonpath-ut + TEST_TARGET + ydb-library-yql-minikql-jsonpath-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-jsonpath-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-minikql-jsonpath-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-minikql-jsonpath-ut + system_allocator +) +vcs_info(ydb-library-yql-minikql-jsonpath-ut) diff --git a/ydb/library/yql/minikql/jsonpath/ut/common_ut.cpp b/ydb/library/yql/minikql/jsonpath/ut/common_ut.cpp index 47a6c68458a..087759f769b 100644 --- a/ydb/library/yql/minikql/jsonpath/ut/common_ut.cpp +++ b/ydb/library/yql/minikql/jsonpath/ut/common_ut.cpp @@ -339,7 +339,7 @@ public: "array": [1, 2, 3, 4] })", "$.array[+$.range.from to +$.range.to]", {"2", "3"}}, {R"([1, 2, 3])", "-$[*]", {"-1", "-2", "-3"}}, - {"10000000000000000000000000", "-$", {"-1e+25"}}, + {"10000000000000000000000000", "-$", {"-9.999999999999999e+24"}}, }; for (const auto& testCase : testCases) { diff --git a/ydb/library/yql/minikql/jsonpath/ya.make b/ydb/library/yql/minikql/jsonpath/ya.make index 629d6246463..68e81ce87e2 100644 --- a/ydb/library/yql/minikql/jsonpath/ya.make +++ b/ydb/library/yql/minikql/jsonpath/ya.make @@ -53,6 +53,10 @@ GENERATE_ENUM_SERIALIZATION(ast_nodes.h) END() +RECURSE( + benchmark +) + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/library/yql/minikql/perf/CMakeLists.txt b/ydb/library/yql/minikql/perf/CMakeLists.txt index 0df202dda16..d9225de1b7d 100644 --- a/ydb/library/yql/minikql/perf/CMakeLists.txt +++ b/ydb/library/yql/minikql/perf/CMakeLists.txt @@ -7,6 +7,8 @@ add_subdirectory(alloc) +add_subdirectory(block_groupby) +add_subdirectory(mprefetch) add_subdirectory(mt_param) add_subdirectory(packer) add_subdirectory(param) diff --git a/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..2363b25d571 --- /dev/null +++ b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,39 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(block_groupby) +target_compile_options(block_groupby PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(block_groupby PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + libs-apache-arrow + yql-minikql-arrow + minikql-comp_nodes-llvm + yql-public-udf + udf-service-exception_policy + yql-sql-pg_dummy + library-cpp-getopt +) +target_link_options(block_groupby PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(block_groupby PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/perf/block_groupby/block_groupby.cpp +) +target_allocator(block_groupby + system_allocator +) +vcs_info(block_groupby) diff --git a/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..5990139b8f4 --- /dev/null +++ b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.linux-aarch64.txt @@ -0,0 +1,42 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(block_groupby) +target_compile_options(block_groupby PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(block_groupby PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + libs-apache-arrow + yql-minikql-arrow + minikql-comp_nodes-llvm + yql-public-udf + udf-service-exception_policy + yql-sql-pg_dummy + library-cpp-getopt +) +target_link_options(block_groupby PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(block_groupby PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/perf/block_groupby/block_groupby.cpp +) +target_allocator(block_groupby + cpp-malloc-jemalloc +) +vcs_info(block_groupby) diff --git a/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..6d662c8956c --- /dev/null +++ b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.linux-x86_64.txt @@ -0,0 +1,44 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(block_groupby) +target_compile_options(block_groupby PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(block_groupby PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + libs-apache-arrow + yql-minikql-arrow + minikql-comp_nodes-llvm + yql-public-udf + udf-service-exception_policy + yql-sql-pg_dummy + library-cpp-getopt +) +target_link_options(block_groupby PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(block_groupby PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/perf/block_groupby/block_groupby.cpp +) +target_allocator(block_groupby + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(block_groupby) diff --git a/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.txt b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..e869124c08f --- /dev/null +++ b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.windows-x86_64.txt @@ -0,0 +1,32 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(block_groupby) +target_compile_options(block_groupby PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(block_groupby PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + libs-apache-arrow + yql-minikql-arrow + minikql-comp_nodes-llvm + yql-public-udf + udf-service-exception_policy + yql-sql-pg_dummy + library-cpp-getopt +) +target_sources(block_groupby PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/perf/block_groupby/block_groupby.cpp +) +target_allocator(block_groupby + system_allocator +) +vcs_info(block_groupby) diff --git a/ydb/library/yql/minikql/perf/block_groupby/block_groupby.cpp b/ydb/library/yql/minikql/perf/block_groupby/block_groupby.cpp new file mode 100644 index 00000000000..a7ee8e3910d --- /dev/null +++ b/ydb/library/yql/minikql/perf/block_groupby/block_groupby.cpp @@ -0,0 +1,497 @@ +#include <util/datetime/cputimer.h> + +#include <ydb/library/yql/minikql/comp_nodes/mkql_rh_hash.h> + +#include <ydb/library/yql/minikql/arrow/arrow_defs.h> +#include <arrow/array/builder_primitive.h> +#include <arrow/datum.h> + +#include <library/cpp/getopt/last_getopt.h> +#include <util/digest/fnv.h> +#include <util/digest/murmur.h> +#include <util/digest/city.h> + +enum class EDistribution { + Const, + Linear, + Random, + Few, + RandomFew +}; + +enum class EShape { + Default, + Sqrt, + Log +}; + +arrow::Datum MakeIntColumn(ui32 len, EDistribution dist, EShape shape, ui32 buckets) { + arrow::Int32Builder builder; + ARROW_OK(builder.Reserve(len)); + for (ui32 i = 0; i < len; ++i) { + ui32 val; + switch (shape) { + case EShape::Default: + val = i; + break; + case EShape::Sqrt: + val = (ui32)sqrt(i); + break; + case EShape::Log: + val = (ui32)log(1 + i); + break; + } + + switch (dist) { + case EDistribution::Const: + builder.UnsafeAppend(0); + break; + case EDistribution::Few: + builder.UnsafeAppend(val % buckets); + break; + case EDistribution::Linear: + builder.UnsafeAppend(val); + break; + case EDistribution::Random: + builder.UnsafeAppend(IntHash(val)); + break; + case EDistribution::RandomFew: + builder.UnsafeAppend(IntHash(val) % buckets); + break; + } + } + + std::shared_ptr<arrow::ArrayData> result; + ARROW_OK(builder.FinishInternal(&result)); + return arrow::Datum(result); +} + +class IAggregator { +public: + virtual ~IAggregator() = default; + virtual void Init(i64* state, i32 payload) = 0; + virtual void Update(i64* state, i32 payload) = 0; +}; + +class TSumAggregator : public IAggregator { +public: + void Init(i64* state, i32 payload) final { + *state = payload; + } + + void Update(i64* state, i32 payload) final { + *state += payload; + } +}; + +template <typename T> +struct TCityHasher { +public: + ui64 operator()(const T& x) const { + return CityHash64(TStringBuf((char*)&x, sizeof(x))); + } +}; + +// sum(payloads) group by keys +template <bool CalculateHashStats, bool UseRH> +class TAggregate { +private: + struct TOneCell { + i32 Key = 0; + bool IsEmpty = true; + i64 State = 0; + }; + + struct TCell { + i32 Key = 0; + i32 PSL = -1; + i64 State = 0; + }; + +public: + TAggregate(const std::vector<IAggregator*>& aggs) + : Aggs(aggs) + , RH(sizeof(i64)) + { + Cells.resize(1u << 8); + } + + void AddBatch(arrow::Datum keys, arrow::Datum payloads) { + auto arrKeys = keys.array(); + auto arrPayloads = payloads.array(); + auto len = arrKeys->length; + const i32* ptrKeys = arrKeys->GetValues<i32>(1); + const i32* ptrPayloads = arrPayloads->GetValues<i32>(1); + for (int64_t i = 0; i < len; ++i) { + auto key = ptrKeys[i]; + auto payload = ptrPayloads[i]; + if (!MoreThanOne) { + if (One.IsEmpty) { + One.IsEmpty = false; + One.Key = key; + for (const auto& a : Aggs) { + a->Init(&One.State, payload); + } + + Size = 1; + continue; + } else { + if (key == One.Key) { + for (const auto& a : Aggs) { + a->Update(&One.State, payload); + } + + continue; + } else { + MoreThanOne = true; + if constexpr (UseRH) { + bool isNew; + auto iter = RH.Insert(One.Key, isNew); + Y_ASSERT(isNew); + *(i64*)RH.GetPayload(iter) = One.State; + } else { + bool isNew; + ui64 bucket = AddBucketFromKeyImpl(One.Key, Cells, isNew); + auto& c = Cells[bucket]; + c.PSL = 0; + c.Key = One.Key; + c.State = One.State; + } + } + } + } + + if constexpr (UseRH) { + bool isNew = false; + auto iter = RH.Insert(key, isNew); + if (isNew) { + for (const auto& a : Aggs) { + a->Init((i64*)RH.GetPayload(iter), payload); + } + + RH.CheckGrow(); + } else { + for (const auto& a : Aggs) { + a->Update((i64*)RH.GetPayload(iter), payload); + } + } + } else { + bool isNew = false; + ui64 bucket = AddBucketFromKey(key, isNew); + auto& c = Cells[bucket]; + if (isNew) { + Size += 1; + for (const auto& a : Aggs) { + a->Init(&c.State, payload); + } + + if (Size * 2 >= Cells.size()) { + Grow(); + } + } else { + for (const auto& a : Aggs) { + a->Update(&c.State, payload); + } + } + } + } + } + + static ui64 MakeHash(i32 key) { + //auto hash = FnvHash<ui64>(&key, sizeof(key)); + //auto hash = MurmurHash<ui64>(&key, sizeof(key)); + auto hash = CityHash64(TStringBuf((char*)&key, sizeof(key))); + //auto hash = key; + return hash; + } + + Y_FORCE_INLINE ui64 AddBucketFromKey(i32 key, bool& isNew) { + return AddBucketFromKeyImpl(key, Cells, isNew); + } + + Y_FORCE_INLINE ui64 AddBucketFromKeyImpl(i32 key, std::vector<TCell>& cells, bool& isNew) { + isNew = false; + ui32 chainLen = 0; + if constexpr (CalculateHashStats) { + HashSearches++; + } + + ui64 bucket = MakeHash(key) & (cells.size() - 1); + i32 distance = 0; + ui64 returnBucket; + i64 oldState; + for (;;) { + if constexpr (CalculateHashStats) { + HashProbes++; + chainLen++; + } + + if (cells[bucket].PSL < 0) { + isNew = true; + cells[bucket].Key = key; + cells[bucket].PSL = distance; + + if constexpr (CalculateHashStats) { + MaxHashChainLen = Max(MaxHashChainLen, chainLen); + } + + return bucket; + } + + if (cells[bucket].Key == key) { + if constexpr (CalculateHashStats) { + MaxHashChainLen = Max(MaxHashChainLen, chainLen); + } + + return bucket; + } + + if (distance > cells[bucket].PSL) { + // swap keys & state + returnBucket = bucket; + oldState = cells[bucket].State; + std::swap(key, cells[bucket].Key); + std::swap(distance, cells[bucket].PSL); + isNew = true; + + ++distance; + bucket = (bucket + 1) & (cells.size() - 1); + break; + } + + ++distance; + bucket = (bucket + 1) & (cells.size() - 1); + } + + for (;;) { + if constexpr (CalculateHashStats) { + HashProbes++; + chainLen++; + } + + if (cells[bucket].PSL < 0) { + if constexpr (CalculateHashStats) { + MaxHashChainLen = Max(MaxHashChainLen, chainLen); + } + + cells[bucket].Key = key; + cells[bucket].State = oldState; + cells[bucket].PSL = distance; + return returnBucket; // for original key + } + + Y_ENSURE(cells[bucket].Key != key); + if (distance > cells[bucket].PSL) { + // swap keys & state + std::swap(key, cells[bucket].Key); + std::swap(oldState, cells[bucket].State); + std::swap(distance, cells[bucket].PSL); + } + + ++distance; + bucket = (bucket + 1) & (cells.size() - 1); + } + } + + void Grow() { + std::vector<TCell> newCells; + newCells.resize(Cells.size() * 2); // must be power of 2 + for (const auto& c : Cells) { + if (c.PSL < 0) { + continue; + } + + bool isNew; + auto newBucket = AddBucketFromKeyImpl(c.Key, newCells, isNew); + auto& nc = newCells[newBucket]; + nc.State = c.State; + } + + Cells.swap(newCells); + } + + double GetAverageHashChainLen() { + return 1.0*HashProbes/HashSearches; + } + + ui32 GetMaxHashChainLen() { + return MaxHashChainLen; + } + + void GetResult(arrow::Datum& keys, arrow::Datum& sums) { + arrow::Int32Builder keysBuilder; + arrow::Int64Builder sumsBuilder; + if (!MoreThanOne) { + if (!One.IsEmpty) { + ARROW_OK(keysBuilder.Reserve(1)); + ARROW_OK(sumsBuilder.Reserve(1)); + keysBuilder.UnsafeAppend(One.Key); + sumsBuilder.UnsafeAppend(One.State); + } + } else { + ui64 size; + if constexpr (UseRH) { + size = RH.GetSize(); + } else { + size = Size; + } + + ARROW_OK(keysBuilder.Reserve(size)); + ARROW_OK(sumsBuilder.Reserve(size)); + i32 maxPSL = 0; + i64 sumPSL = 0; + if constexpr (UseRH) { + for (auto iter = RH.Begin(); iter != RH.End(); RH.Advance(iter)) { + auto& psl = RH.GetPSL(iter); + if (psl.Distance < 0) { + continue; + } + + keysBuilder.UnsafeAppend(RH.GetKey(iter)); + sumsBuilder.UnsafeAppend(*(i64*)RH.GetPayload(iter)); + maxPSL = Max(psl.Distance, maxPSL); + sumPSL += psl.Distance; + } + } else { + for (const auto& c : Cells) { + if (c.PSL < 0) { + continue; + } + + keysBuilder.UnsafeAppend(c.Key); + sumsBuilder.UnsafeAppend(c.State); + maxPSL = Max(c.PSL, maxPSL); + sumPSL += c.PSL; + } + } + + if constexpr (CalculateHashStats) { + Cerr << "maxPSL = " << maxPSL << "\n"; + Cerr << "avgPSL = " << 1.0*sumPSL/size << "\n"; + } + } + + std::shared_ptr<arrow::ArrayData> keysData; + ARROW_OK(keysBuilder.FinishInternal(&keysData)); + keys = keysData; + + std::shared_ptr<arrow::ArrayData> sumsData; + ARROW_OK(sumsBuilder.FinishInternal(&sumsData)); + sums = sumsData; + } + +private: + bool MoreThanOne = false; + TOneCell One; + std::vector<TCell> Cells; + ui64 Size = 0; + + const std::vector<IAggregator*> Aggs; + ui64 HashProbes = 0; + ui64 HashSearches = 0; + ui32 MaxHashChainLen = 0; + + NKikimr::NMiniKQL::TRobinHoodHashMap<i32> RH; + NKikimr::NMiniKQL::TRobinHoodHashSet<i32> RHS; +}; + +int main(int argc, char** argv) { + NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default(); + TString keysDistributionStr; + TString shapeStr="default"; + ui32 nIters = 100; + ui32 nRows = 1000000; + ui32 nBuckets = 16; + ui32 nRepeats = 10; + opts.AddLongOption('k', "keys", "distribution of keys (const, linear, random, few, randomfew)").StoreResult(&keysDistributionStr).Required(); + opts.AddLongOption('s', "shape", "shape of counter (default, sqrt, log)").StoreResult(&shapeStr); + opts.AddLongOption('i', "iter", "# of iterations").StoreResult(&nIters); + opts.AddLongOption('r', "rows", "# of rows").StoreResult(&nRows); + opts.AddLongOption('b', "buckets", "modulo for few/randomfew").StoreResult(&nBuckets); + opts.AddLongOption('t', "repeats", "# of repeats").StoreResult(&nRepeats); + opts.SetFreeArgsMax(0); + NLastGetopt::TOptsParseResult res(&opts, argc, argv); + EDistribution keysDist; + EShape shape = EShape::Default; + if (keysDistributionStr == "const") { + keysDist = EDistribution::Const; + } else if (keysDistributionStr == "linear") { + keysDist = EDistribution::Linear; + } else if (keysDistributionStr == "random") { + keysDist = EDistribution::Random; + } else if (keysDistributionStr == "few") { + keysDist = EDistribution::Few; + } else if (keysDistributionStr == "randomfew") { + keysDist = EDistribution::RandomFew; + } else { + ythrow yexception() << "Unsupported distribution: " << keysDistributionStr; + } + + if (shapeStr == "default") { + shape = EShape::Default; + } else if (shapeStr == "sqrt") { + shape = EShape::Sqrt; + } else if (shapeStr == "log") { + shape = EShape::Log; + } else { + ythrow yexception() << "Unsupported shape: " << shapeStr; + } + + auto col1 = MakeIntColumn(nRows, keysDist, shape, nBuckets); + auto col2 = MakeIntColumn(nRows, EDistribution::Linear, EShape::Default, nBuckets); + Cerr << "col1.length: " << col1.length() << "\n"; + Cerr << "col2.length: " << col2.length() << "\n"; + + TSumAggregator sum; + std::vector<IAggregator*> aggs; + aggs.push_back(&sum); + TAggregate<true, true> agg(aggs); + agg.AddBatch(col1, col2); + arrow::Datum keys, sums; + agg.GetResult(keys, sums); + ui64 total1 = 0; + for (ui32 i = 0; i < col2.length(); ++i) { + total1 += col2.array()->GetValues<i32>(1)[i]; + } + + Cerr << "total1: " << total1 << "\n"; + ui64 total2 = 0; + + Cerr << "keys.length: " << keys.length() << "\n"; + Cerr << "sums.length: " << sums.length() << "\n"; + for (ui32 i = 0; i < sums.length(); ++i) { + total2 += sums.array()->GetValues<i64>(1)[i]; + } + + Cerr << "total2: " << total2 << "\n"; + Y_ENSURE(total1 == total2); + Cerr << "AverageHashChainLen: " << agg.GetAverageHashChainLen() << "\n"; + Cerr << "MaxHashChainLen: " << agg.GetMaxHashChainLen() << "\n"; + + std::vector<double> durations; + for (ui32 j = 0; j < nRepeats; ++j) { + TSimpleTimer timer; + for (ui32 i = 0; i < nIters; ++i) { + TAggregate<false, true> agg(aggs); + agg.AddBatch(col1, col2); + arrow::Datum keys, sums; + agg.GetResult(keys, sums); + } + + auto duration = timer.Get(); + durations.push_back(1e-6*duration.MicroSeconds()); + } + + double sumDurations = 0.0, sumDurationsQ = 0.0; + for (auto d : durations) { + sumDurations += d; + sumDurationsQ += d * d; + } + + double avgDuration = sumDurations / nRepeats; + double dispDuration = sqrt(sumDurationsQ / nRepeats - avgDuration * avgDuration); + Cerr << "Elapsed: " << avgDuration << ", noise: " << 100*dispDuration/avgDuration << "%\n"; + Cerr << "Speed: " << 1e-6 * (ui64(nIters) * nRows / avgDuration) << " M rows/sec\n"; + Cerr << "Speed: " << 1e-6 * (2 * sizeof(i32) * ui64(nIters) * nRows / avgDuration) << " M bytes/sec\n"; + return 0; +} diff --git a/ydb/library/yql/minikql/perf/block_groupby/ya.make b/ydb/library/yql/minikql/perf/block_groupby/ya.make new file mode 100644 index 00000000000..29e7fd95ca1 --- /dev/null +++ b/ydb/library/yql/minikql/perf/block_groupby/ya.make @@ -0,0 +1,19 @@ +PROGRAM() + +PEERDIR( + contrib/libs/apache/arrow + ydb/library/yql/minikql/arrow + ydb/library/yql/minikql/comp_nodes/llvm + ydb/library/yql/public/udf + ydb/library/yql/public/udf/service/exception_policy + ydb/library/yql/sql/pg_dummy + library/cpp/getopt +) + +SRCS( + block_groupby.cpp +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..d0b1af3792f --- /dev/null +++ b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,32 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(mprefetch) +target_compile_options(mprefetch PRIVATE + -mprfchw + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(mprefetch PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + library-cpp-getopt +) +target_link_options(mprefetch PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC +) +target_sources(mprefetch PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/perf/mprefetch/mprefetch.cpp +) +target_allocator(mprefetch + system_allocator +) +vcs_info(mprefetch) diff --git a/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..318241f220e --- /dev/null +++ b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.linux-aarch64.txt @@ -0,0 +1,37 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(mprefetch) +target_compile_options(mprefetch PRIVATE + -mprfchw + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(mprefetch PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-getopt +) +target_link_options(mprefetch PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(mprefetch PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/perf/mprefetch/mprefetch.cpp +) +target_allocator(mprefetch + cpp-malloc-jemalloc +) +vcs_info(mprefetch) diff --git a/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..bd71fd7c81f --- /dev/null +++ b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.linux-x86_64.txt @@ -0,0 +1,39 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(mprefetch) +target_compile_options(mprefetch PRIVATE + -mprfchw + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(mprefetch PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + library-cpp-getopt +) +target_link_options(mprefetch PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(mprefetch PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/perf/mprefetch/mprefetch.cpp +) +target_allocator(mprefetch + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(mprefetch) diff --git a/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.txt b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..f2f199b2400 --- /dev/null +++ b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.windows-x86_64.txt @@ -0,0 +1,27 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(mprefetch) +target_compile_options(mprefetch PRIVATE + -mprfchw + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(mprefetch PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + library-cpp-getopt +) +target_sources(mprefetch PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/perf/mprefetch/mprefetch.cpp +) +target_allocator(mprefetch + system_allocator +) +vcs_info(mprefetch) diff --git a/ydb/library/yql/minikql/perf/mprefetch/mprefetch.cpp b/ydb/library/yql/minikql/perf/mprefetch/mprefetch.cpp new file mode 100644 index 00000000000..9ba9cbc3df4 --- /dev/null +++ b/ydb/library/yql/minikql/perf/mprefetch/mprefetch.cpp @@ -0,0 +1,108 @@ +#include <util/datetime/cputimer.h> + +#include <library/cpp/getopt/last_getopt.h> + +#include <deque> + +int main(int argc, char** argv) { + NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default(); + ui32 nIters = 100000000; + ui32 nRows = 100000000; + ui32 nRepeats = 10; + ui32 nPrefetch = 0; + ui32 nSpin = 0; + bool shuffle = true; + opts.AddLongOption('r', "rows", "# of rows").StoreResult(&nRows); + opts.AddLongOption('i', "iter", "# of iterations").StoreResult(&nIters); + opts.AddLongOption('t', "repeats", "# of repeats").StoreResult(&nRepeats); + opts.AddLongOption('p', "prefetch", "# of prefetch").StoreResult(&nPrefetch); + opts.AddLongOption('h', "shuffle", "randomize").StoreResult(&shuffle); + opts.AddLongOption('s', "spin", "spin count").StoreResult(&nSpin); + opts.SetFreeArgsMax(0); + NLastGetopt::TOptsParseResult res(&opts, argc, argv); + + std::vector<ui32> v(nRows); + std::vector<ui32> data(nRows); + std::iota(v.begin(), v.end(), 0); + if (shuffle) { + std::random_shuffle(v.begin(), v.end()); + } + + std::vector<ui32> prefetchQueue(nPrefetch); + ui32 queueBegin = 0; + ui32 queueEnd = 0; + ui32 queueSize = 0; + volatile ui64 tmp = 0; + std::vector<double> durations; + for (ui32 j = 0; j < nRepeats; ++j) { + TSimpleTimer timer; + ui32 index = 0; + if (nPrefetch == 0) { + for (ui32 i = 0; i < nIters; ++i) { + data[v[index++]]+=1; + if (index == nRows) { + index = 0; + } + + for (ui32 j = 0; j < nSpin; ++j) { + ++tmp; + } + } + } else { + auto handle = [&]() { + auto prevJ = prefetchQueue[queueBegin++]; + --queueSize; + if (queueBegin == nPrefetch) { + queueBegin = 0; + } + + data[prevJ]+=1; + + for (ui32 j = 0; j < nSpin; ++j) { + ++tmp; + } + }; + + for (ui32 i = 0; i < nIters; ++i) { + auto j = v[index++]; + if (index == nRows) { + index = 0; + } + + __builtin_prefetch(data.data() + j, 1, 3); + prefetchQueue[queueEnd++] = j; + ++queueSize; + if (queueEnd == nPrefetch) { + queueEnd = 0; + } + + if (queueSize == nPrefetch) { + handle(); + } + } + + while (queueSize > 0) { + handle(); + } + } + + auto duration = timer.Get(); + durations.push_back(1e-6*duration.MicroSeconds()); + } + + // remove 1/3 of worst measurements + Sort(durations.begin(), durations.end()); + durations.erase(durations.begin() + nRepeats * 2 / 3, durations.end()); + nRepeats = durations.size(); + double sumDurations = 0.0, sumDurationsQ = 0.0; + for (auto d : durations) { + sumDurations += d; + sumDurationsQ += d * d; + } + + double avgDuration = sumDurations / nRepeats; + double dispDuration = sqrt(sumDurationsQ / nRepeats - avgDuration * avgDuration); + Cerr << "Elapsed: " << avgDuration << ", noise: " << 100*dispDuration/avgDuration << "%\n"; + Cerr << "Speed: " << 1e-6 * (ui64(nIters) / avgDuration) << " M iters/sec\n"; + return 0; +} diff --git a/ydb/library/yql/minikql/perf/mprefetch/ya.make b/ydb/library/yql/minikql/perf/mprefetch/ya.make new file mode 100644 index 00000000000..c7d873f7237 --- /dev/null +++ b/ydb/library/yql/minikql/perf/mprefetch/ya.make @@ -0,0 +1,17 @@ +PROGRAM() + +PEERDIR( + library/cpp/getopt +) + +SRCS( + mprefetch.cpp +) + +CFLAGS( + -mprfchw +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/library/yql/minikql/perf/ya.make b/ydb/library/yql/minikql/perf/ya.make new file mode 100644 index 00000000000..dd83a764f9a --- /dev/null +++ b/ydb/library/yql/minikql/perf/ya.make @@ -0,0 +1,10 @@ + +RECURSE( + alloc + block_groupby + mprefetch + mt_param + packer + param + presort +) diff --git a/ydb/library/yql/minikql/ya.make b/ydb/library/yql/minikql/ya.make index fd1ef197f08..8c64d9eb3c7 100644 --- a/ydb/library/yql/minikql/ya.make +++ b/ydb/library/yql/minikql/ya.make @@ -83,6 +83,19 @@ YQL_LAST_ABI_VERSION() END() +RECURSE( + arrow + benchmark + codegen + comp_nodes + computation + datetime + dom + invoke_builtins + jsonpath + perf +) + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/library/yql/parser/proto_ast/ya.make b/ydb/library/yql/parser/proto_ast/ya.make index fb7290cadb5..8251c29c182 100644 --- a/ydb/library/yql/parser/proto_ast/ya.make +++ b/ydb/library/yql/parser/proto_ast/ya.make @@ -12,6 +12,6 @@ SRCS( END() RECURSE( - gen collect_issues + gen ) diff --git a/ydb/library/yql/providers/clickhouse/ya.make b/ydb/library/yql/providers/clickhouse/ya.make index eacab82215a..58d9016b198 100644 --- a/ydb/library/yql/providers/clickhouse/ya.make +++ b/ydb/library/yql/providers/clickhouse/ya.make @@ -1,6 +1,6 @@ RECURSE( actors expr_nodes - provider proto + provider ) diff --git a/ydb/library/yql/providers/common/arrow/ya.make b/ydb/library/yql/providers/common/arrow/ya.make index 5c032323097..328794fdd57 100644 --- a/ydb/library/yql/providers/common/arrow/ya.make +++ b/ydb/library/yql/providers/common/arrow/ya.make @@ -14,4 +14,8 @@ CFLAGS( -DARCADIA_BUILD -DUSE_PARQUET ) -END()
\ No newline at end of file +END() + +RECURSE( + interface +) diff --git a/ydb/library/yql/providers/common/metrics/ya.make b/ydb/library/yql/providers/common/metrics/ya.make index c40f275be35..8b54fc73a0d 100644 --- a/ydb/library/yql/providers/common/metrics/ya.make +++ b/ydb/library/yql/providers/common/metrics/ya.make @@ -13,3 +13,7 @@ PEERDIR( ) END() + +RECURSE( + protos +) diff --git a/ydb/library/yql/providers/common/proto/python/ya.make b/ydb/library/yql/providers/common/proto/python/ya.make new file mode 100644 index 00000000000..fc9cc6f5b91 --- /dev/null +++ b/ydb/library/yql/providers/common/proto/python/ya.make @@ -0,0 +1,2 @@ +PY_PROTOS_FOR(ydb/library/yql/providers/common/proto) + diff --git a/ydb/library/yql/providers/common/schema/ya.make b/ydb/library/yql/providers/common/schema/ya.make index 4357f787d5d..f807ad3fe26 100644 --- a/ydb/library/yql/providers/common/schema/ya.make +++ b/ydb/library/yql/providers/common/schema/ya.make @@ -10,3 +10,10 @@ PEERDIR( ) END() + +RECURSE( + expr + mkql + parser + skiff +) diff --git a/ydb/library/yql/providers/common/ya.make b/ydb/library/yql/providers/common/ya.make index 275a09ad198..7bef7eaedda 100644 --- a/ydb/library/yql/providers/common/ya.make +++ b/ydb/library/yql/providers/common/ya.make @@ -1,8 +1,9 @@ RECURSE( activation - arrow_resolve arrow + arrow_resolve codec + comp_nodes config db_id_async_resolver dq @@ -10,14 +11,10 @@ RECURSE( http_gateway metrics mkql - comp_nodes proto + proto/python provider schema - schema/expr - schema/mkql - schema/parser - schema/skiff structured_token token_accessor transform diff --git a/ydb/library/yql/providers/dq/provider/ya.make b/ydb/library/yql/providers/dq/provider/ya.make index 5765c684d44..aec22bc2bdc 100644 --- a/ydb/library/yql/providers/dq/provider/ya.make +++ b/ydb/library/yql/providers/dq/provider/ya.make @@ -61,6 +61,10 @@ YQL_LAST_ABI_VERSION() END() +RECURSE( + exec +) + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/library/yql/providers/dq/worker_manager/ya.make b/ydb/library/yql/providers/dq/worker_manager/ya.make index 63c61eabd8d..8ef4840b723 100644 --- a/ydb/library/yql/providers/dq/worker_manager/ya.make +++ b/ydb/library/yql/providers/dq/worker_manager/ya.make @@ -22,3 +22,7 @@ SRCS( ) END() + +RECURSE( + interface +) diff --git a/ydb/library/yql/providers/dq/ya.make b/ydb/library/yql/providers/dq/ya.make index f4af5913426..e354a8a22f4 100644 --- a/ydb/library/yql/providers/dq/ya.make +++ b/ydb/library/yql/providers/dq/ya.make @@ -10,7 +10,6 @@ RECURSE( opt planner provider - provider/exec runtime task_runner task_runner_actor diff --git a/ydb/library/yql/providers/generic/ya.make b/ydb/library/yql/providers/generic/ya.make index 26a0d87ba62..5e9adf2bc83 100644 --- a/ydb/library/yql/providers/generic/ya.make +++ b/ydb/library/yql/providers/generic/ya.make @@ -2,8 +2,8 @@ RECURSE( actors connector expr_nodes - provider proto + provider ) RECURSE_FOR_TESTS( diff --git a/ydb/library/yql/providers/s3/object_listers/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/s3/object_listers/CMakeLists.darwin-x86_64.txt index 5b00b830ac8..ad94789cb73 100644 --- a/ydb/library/yql/providers/s3/object_listers/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/providers/s3/object_listers/CMakeLists.darwin-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/providers/s3/object_listers/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/s3/object_listers/CMakeLists.linux-aarch64.txt index 0155191e4bf..abf0a0e4d69 100644 --- a/ydb/library/yql/providers/s3/object_listers/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/providers/s3/object_listers/CMakeLists.linux-aarch64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/providers/s3/object_listers/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/s3/object_listers/CMakeLists.linux-x86_64.txt index 0155191e4bf..abf0a0e4d69 100644 --- a/ydb/library/yql/providers/s3/object_listers/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/providers/s3/object_listers/CMakeLists.linux-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/providers/s3/object_listers/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/s3/object_listers/CMakeLists.windows-x86_64.txt index 5b00b830ac8..ad94789cb73 100644 --- a/ydb/library/yql/providers/s3/object_listers/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/providers/s3/object_listers/CMakeLists.windows-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..8cb595c1e1c --- /dev/null +++ b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,67 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-providers-s3-object_listers-ut) +target_include_directories(ydb-library-yql-providers-s3-object_listers-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/object_listers +) +target_link_libraries(ydb-library-yql-providers-s3-object_listers-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + providers-s3-object_listers +) +target_link_options(ydb-library-yql-providers-s3-object_listers-ut PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-library-yql-providers-s3-object_listers-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/object_listers/yql_s3_path_ut.cpp +) +set_property( + TARGET + ydb-library-yql-providers-s3-object_listers-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-providers-s3-object_listers-ut + TEST_TARGET + ydb-library-yql-providers-s3-object_listers-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-providers-s3-object_listers-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-providers-s3-object_listers-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-providers-s3-object_listers-ut + system_allocator +) +vcs_info(ydb-library-yql-providers-s3-object_listers-ut) diff --git a/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..401b4507243 --- /dev/null +++ b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.linux-aarch64.txt @@ -0,0 +1,70 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-providers-s3-object_listers-ut) +target_include_directories(ydb-library-yql-providers-s3-object_listers-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/object_listers +) +target_link_libraries(ydb-library-yql-providers-s3-object_listers-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + providers-s3-object_listers +) +target_link_options(ydb-library-yql-providers-s3-object_listers-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-providers-s3-object_listers-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/object_listers/yql_s3_path_ut.cpp +) +set_property( + TARGET + ydb-library-yql-providers-s3-object_listers-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-providers-s3-object_listers-ut + TEST_TARGET + ydb-library-yql-providers-s3-object_listers-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-providers-s3-object_listers-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-providers-s3-object_listers-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-providers-s3-object_listers-ut + cpp-malloc-jemalloc +) +vcs_info(ydb-library-yql-providers-s3-object_listers-ut) diff --git a/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..cbb021028ba --- /dev/null +++ b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.linux-x86_64.txt @@ -0,0 +1,72 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-providers-s3-object_listers-ut) +target_include_directories(ydb-library-yql-providers-s3-object_listers-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/object_listers +) +target_link_libraries(ydb-library-yql-providers-s3-object_listers-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + providers-s3-object_listers +) +target_link_options(ydb-library-yql-providers-s3-object_listers-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-providers-s3-object_listers-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/object_listers/yql_s3_path_ut.cpp +) +set_property( + TARGET + ydb-library-yql-providers-s3-object_listers-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-providers-s3-object_listers-ut + TEST_TARGET + ydb-library-yql-providers-s3-object_listers-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-providers-s3-object_listers-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-providers-s3-object_listers-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-providers-s3-object_listers-ut + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(ydb-library-yql-providers-s3-object_listers-ut) diff --git a/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.txt b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..eeeb92a6ea2 --- /dev/null +++ b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.windows-x86_64.txt @@ -0,0 +1,60 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-providers-s3-object_listers-ut) +target_include_directories(ydb-library-yql-providers-s3-object_listers-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/object_listers +) +target_link_libraries(ydb-library-yql-providers-s3-object_listers-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + providers-s3-object_listers +) +target_sources(ydb-library-yql-providers-s3-object_listers-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/object_listers/yql_s3_path_ut.cpp +) +set_property( + TARGET + ydb-library-yql-providers-s3-object_listers-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-providers-s3-object_listers-ut + TEST_TARGET + ydb-library-yql-providers-s3-object_listers-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-providers-s3-object_listers-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-providers-s3-object_listers-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-providers-s3-object_listers-ut + system_allocator +) +vcs_info(ydb-library-yql-providers-s3-object_listers-ut) diff --git a/ydb/library/yql/providers/s3/ya.make b/ydb/library/yql/providers/s3/ya.make index 6408970d3b3..0f22cf790ee 100644 --- a/ydb/library/yql/providers/s3/ya.make +++ b/ydb/library/yql/providers/s3/ya.make @@ -3,6 +3,7 @@ RECURSE( common compressors expr_nodes + object_listers path_generator proto provider diff --git a/ydb/library/yql/providers/stat/ya.make b/ydb/library/yql/providers/stat/ya.make new file mode 100644 index 00000000000..fc76f599d96 --- /dev/null +++ b/ydb/library/yql/providers/stat/ya.make @@ -0,0 +1,5 @@ +RECURSE( + expr_nodes + uploader +) + diff --git a/ydb/library/yql/providers/ya.make b/ydb/library/yql/providers/ya.make index 5366e2c7d8f..bae71fcd1e6 100644 --- a/ydb/library/yql/providers/ya.make +++ b/ydb/library/yql/providers/ya.make @@ -3,13 +3,14 @@ RECURSE( common config dq + function + generic pq result s3 solomon + stat ydb yt - function - generic ) diff --git a/ydb/library/yql/providers/yt/codec/ya.make b/ydb/library/yql/providers/yt/codec/ya.make index ce64d963adf..558538625b4 100644 --- a/ydb/library/yql/providers/yt/codec/ya.make +++ b/ydb/library/yql/providers/yt/codec/ya.make @@ -40,6 +40,10 @@ YQL_LAST_ABI_VERSION() END() +RECURSE( + codegen +) + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/library/yql/providers/yt/comp_nodes/ya.make b/ydb/library/yql/providers/yt/comp_nodes/ya.make index b1d3159abd2..5b950ff88b7 100644 --- a/ydb/library/yql/providers/yt/comp_nodes/ya.make +++ b/ydb/library/yql/providers/yt/comp_nodes/ya.make @@ -25,6 +25,10 @@ YQL_LAST_ABI_VERSION() END() +RECURSE( + dq +) + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/library/yql/providers/yt/lib/ya.make b/ydb/library/yql/providers/yt/lib/ya.make index 9fb3a47e1c9..108cc9d99bd 100644 --- a/ydb/library/yql/providers/yt/lib/ya.make +++ b/ydb/library/yql/providers/yt/lib/ya.make @@ -1,4 +1,5 @@ RECURSE( + config_clusters expr_traits graph_reorder hash @@ -14,6 +15,6 @@ RECURSE( skiff url_mapper yson_helpers - yt_url_lister yt_download + yt_url_lister ) diff --git a/ydb/library/yql/providers/yt/ya.make b/ydb/library/yql/providers/yt/ya.make index a43252062fa..1a788e63871 100644 --- a/ydb/library/yql/providers/yt/ya.make +++ b/ydb/library/yql/providers/yt/ya.make @@ -1,11 +1,7 @@ RECURSE( codec - codec/codegen/ut - codec/ut common comp_nodes - comp_nodes/ut - comp_nodes/dq expr_nodes gateway job @@ -13,5 +9,4 @@ RECURSE( mkql_dq opt provider - provider/ut ) diff --git a/ydb/library/yql/public/issue/ya.make b/ydb/library/yql/public/issue/ya.make index 65441b9531c..8d7ad13b1c6 100644 --- a/ydb/library/yql/public/issue/ya.make +++ b/ydb/library/yql/public/issue/ya.make @@ -21,6 +21,10 @@ GENERATE_ENUM_SERIALIZATION(yql_warning.h) END() +RECURSE( + protos +) + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/library/yql/public/purecalc/examples/protobuf_pull_list/ya.make b/ydb/library/yql/public/purecalc/examples/protobuf_pull_list/ya.make index 953ff1bf921..cf800933af6 100644 --- a/ydb/library/yql/public/purecalc/examples/protobuf_pull_list/ya.make +++ b/ydb/library/yql/public/purecalc/examples/protobuf_pull_list/ya.make @@ -11,9 +11,7 @@ PEERDIR( ydb/library/yql/public/purecalc/helpers/stream ) - - YQL_LAST_ABI_VERSION() - +YQL_LAST_ABI_VERSION() END() diff --git a/ydb/library/yql/public/purecalc/examples/ya.make b/ydb/library/yql/public/purecalc/examples/ya.make index ad5853e9c49..d78f8a825d0 100644 --- a/ydb/library/yql/public/purecalc/examples/ya.make +++ b/ydb/library/yql/public/purecalc/examples/ya.make @@ -1,7 +1,5 @@ RECURSE( protobuf - protobuf/ut protobuf_pull_list - protobuf_pull_list/ut skiff_pull_list ) diff --git a/ydb/library/yql/public/purecalc/io_specs/protobuf/ya.make b/ydb/library/yql/public/purecalc/io_specs/protobuf/ya.make index a9efad989f5..7177024003f 100644 --- a/ydb/library/yql/public/purecalc/io_specs/protobuf/ya.make +++ b/ydb/library/yql/public/purecalc/io_specs/protobuf/ya.make @@ -10,9 +10,7 @@ SRCS( proto_variant.cpp ) - - YQL_LAST_ABI_VERSION() - +YQL_LAST_ABI_VERSION() END() diff --git a/ydb/library/yql/public/purecalc/ya.make b/ydb/library/yql/public/purecalc/ya.make index 3ac6ab6b59a..cb99c1d3111 100644 --- a/ydb/library/yql/public/purecalc/ya.make +++ b/ydb/library/yql/public/purecalc/ya.make @@ -13,7 +13,13 @@ YQL_LAST_ABI_VERSION() END() +RECURSE( + common + examples + helpers + io_specs +) + RECURSE_FOR_TESTS( - io_specs/ut ut ) diff --git a/ydb/library/yql/public/udf/service/ya.make b/ydb/library/yql/public/udf/service/ya.make new file mode 100644 index 00000000000..e03927ed91b --- /dev/null +++ b/ydb/library/yql/public/udf/service/ya.make @@ -0,0 +1,5 @@ +RECURSE( + exception_policy + stub + terminate_policy +) diff --git a/ydb/library/yql/public/udf/tz/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/public/udf/tz/CMakeLists.darwin-x86_64.txt index 34a38599296..764f26f8b15 100644 --- a/ydb/library/yql/public/udf/tz/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/public/udf/tz/CMakeLists.darwin-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) add_library(public-udf-tz) target_link_libraries(public-udf-tz PUBLIC diff --git a/ydb/library/yql/public/udf/tz/CMakeLists.linux-aarch64.txt b/ydb/library/yql/public/udf/tz/CMakeLists.linux-aarch64.txt index 324920a3a14..85f78011a68 100644 --- a/ydb/library/yql/public/udf/tz/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/public/udf/tz/CMakeLists.linux-aarch64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) add_library(public-udf-tz) target_link_libraries(public-udf-tz PUBLIC diff --git a/ydb/library/yql/public/udf/tz/CMakeLists.linux-x86_64.txt b/ydb/library/yql/public/udf/tz/CMakeLists.linux-x86_64.txt index 324920a3a14..85f78011a68 100644 --- a/ydb/library/yql/public/udf/tz/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/public/udf/tz/CMakeLists.linux-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) add_library(public-udf-tz) target_link_libraries(public-udf-tz PUBLIC diff --git a/ydb/library/yql/public/udf/tz/CMakeLists.windows-x86_64.txt b/ydb/library/yql/public/udf/tz/CMakeLists.windows-x86_64.txt index 34a38599296..764f26f8b15 100644 --- a/ydb/library/yql/public/udf/tz/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/public/udf/tz/CMakeLists.windows-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) add_library(public-udf-tz) target_link_libraries(public-udf-tz PUBLIC diff --git a/ydb/library/yql/public/udf/tz/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/public/udf/tz/ut/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..a26f52dd986 --- /dev/null +++ b/ydb/library/yql/public/udf/tz/ut/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,65 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-public-udf-tz-ut) +target_include_directories(ydb-library-yql-public-udf-tz-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/public/udf/tz +) +target_link_libraries(ydb-library-yql-public-udf-tz-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + public-udf-tz +) +target_link_options(ydb-library-yql-public-udf-tz-ut PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC +) +target_sources(ydb-library-yql-public-udf-tz-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/public/udf/tz/udf_tz_ut.cpp +) +set_property( + TARGET + ydb-library-yql-public-udf-tz-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-public-udf-tz-ut + TEST_TARGET + ydb-library-yql-public-udf-tz-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-public-udf-tz-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-public-udf-tz-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-public-udf-tz-ut + system_allocator +) +vcs_info(ydb-library-yql-public-udf-tz-ut) diff --git a/ydb/library/yql/public/udf/tz/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/public/udf/tz/ut/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..de01319ee97 --- /dev/null +++ b/ydb/library/yql/public/udf/tz/ut/CMakeLists.linux-aarch64.txt @@ -0,0 +1,70 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-public-udf-tz-ut) +target_include_directories(ydb-library-yql-public-udf-tz-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/public/udf/tz +) +target_link_libraries(ydb-library-yql-public-udf-tz-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + public-udf-tz +) +target_link_options(ydb-library-yql-public-udf-tz-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-public-udf-tz-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/public/udf/tz/udf_tz_ut.cpp +) +set_property( + TARGET + ydb-library-yql-public-udf-tz-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-public-udf-tz-ut + TEST_TARGET + ydb-library-yql-public-udf-tz-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-public-udf-tz-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-public-udf-tz-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-public-udf-tz-ut + cpp-malloc-jemalloc +) +vcs_info(ydb-library-yql-public-udf-tz-ut) diff --git a/ydb/library/yql/public/udf/tz/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/public/udf/tz/ut/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..d7f98a2a6c2 --- /dev/null +++ b/ydb/library/yql/public/udf/tz/ut/CMakeLists.linux-x86_64.txt @@ -0,0 +1,72 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-public-udf-tz-ut) +target_include_directories(ydb-library-yql-public-udf-tz-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/public/udf/tz +) +target_link_libraries(ydb-library-yql-public-udf-tz-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + public-udf-tz +) +target_link_options(ydb-library-yql-public-udf-tz-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-public-udf-tz-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/public/udf/tz/udf_tz_ut.cpp +) +set_property( + TARGET + ydb-library-yql-public-udf-tz-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-public-udf-tz-ut + TEST_TARGET + ydb-library-yql-public-udf-tz-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-public-udf-tz-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-public-udf-tz-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-public-udf-tz-ut + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(ydb-library-yql-public-udf-tz-ut) diff --git a/ydb/library/yql/public/udf/tz/ut/CMakeLists.txt b/ydb/library/yql/public/udf/tz/ut/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/public/udf/tz/ut/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/public/udf/tz/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/public/udf/tz/ut/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..fa8472deae7 --- /dev/null +++ b/ydb/library/yql/public/udf/tz/ut/CMakeLists.windows-x86_64.txt @@ -0,0 +1,60 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-public-udf-tz-ut) +target_include_directories(ydb-library-yql-public-udf-tz-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/public/udf/tz +) +target_link_libraries(ydb-library-yql-public-udf-tz-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + public-udf-tz +) +target_sources(ydb-library-yql-public-udf-tz-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/public/udf/tz/udf_tz_ut.cpp +) +set_property( + TARGET + ydb-library-yql-public-udf-tz-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-public-udf-tz-ut + TEST_TARGET + ydb-library-yql-public-udf-tz-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-public-udf-tz-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-public-udf-tz-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-public-udf-tz-ut + system_allocator +) +vcs_info(ydb-library-yql-public-udf-tz-ut) diff --git a/ydb/library/yql/public/udf/tz/ya.make b/ydb/library/yql/public/udf/tz/ya.make index 361d9d77915..5cf803181da 100644 --- a/ydb/library/yql/public/udf/tz/ya.make +++ b/ydb/library/yql/public/udf/tz/ya.make @@ -12,6 +12,10 @@ PROVIDES(YqlUdfTz) END() +RECURSE( + gen +) + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/library/yql/public/udf/ya.make b/ydb/library/yql/public/udf/ya.make index 27a6eed2555..a26635cf9ba 100644 --- a/ydb/library/yql/public/udf/ya.make +++ b/ydb/library/yql/public/udf/ya.make @@ -53,6 +53,13 @@ PROVIDES(YqlUdfSdk) END() +RECURSE( + arrow + service + support + tz +) + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/library/yql/public/ya.make b/ydb/library/yql/public/ya.make index bdbac7a8202..676f60c6b3a 100644 --- a/ydb/library/yql/public/ya.make +++ b/ydb/library/yql/public/ya.make @@ -4,14 +4,6 @@ RECURSE( fastcheck issue purecalc - purecalc/examples - purecalc/helpers - purecalc/io_specs types udf - udf/arrow - udf/service/exception_policy - udf/service/stub - udf/service/terminate_policy - udf/tz/gen ) diff --git a/ydb/library/yql/sql/v0/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/sql/v0/CMakeLists.darwin-x86_64.txt index f2ee61e6f4f..3a805a5ebca 100644 --- a/ydb/library/yql/sql/v0/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/sql/v0/CMakeLists.darwin-x86_64.txt @@ -7,6 +7,7 @@ add_subdirectory(lexer) +add_subdirectory(ut) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/sql/v0/CMakeLists.linux-aarch64.txt b/ydb/library/yql/sql/v0/CMakeLists.linux-aarch64.txt index 350fbe3fcc7..52c5ad75fc2 100644 --- a/ydb/library/yql/sql/v0/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/sql/v0/CMakeLists.linux-aarch64.txt @@ -7,6 +7,7 @@ add_subdirectory(lexer) +add_subdirectory(ut) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/sql/v0/CMakeLists.linux-x86_64.txt b/ydb/library/yql/sql/v0/CMakeLists.linux-x86_64.txt index 350fbe3fcc7..52c5ad75fc2 100644 --- a/ydb/library/yql/sql/v0/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/sql/v0/CMakeLists.linux-x86_64.txt @@ -7,6 +7,7 @@ add_subdirectory(lexer) +add_subdirectory(ut) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/sql/v0/CMakeLists.windows-x86_64.txt b/ydb/library/yql/sql/v0/CMakeLists.windows-x86_64.txt index f2ee61e6f4f..3a805a5ebca 100644 --- a/ydb/library/yql/sql/v0/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/sql/v0/CMakeLists.windows-x86_64.txt @@ -7,6 +7,7 @@ add_subdirectory(lexer) +add_subdirectory(ut) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/sql/v0/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/sql/v0/ut/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..dab39fc9adb --- /dev/null +++ b/ydb/library/yql/sql/v0/ut/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,77 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-sql-v0-ut) +target_include_directories(ydb-library-yql-sql-v0-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v0 +) +target_link_libraries(ydb-library-yql-sql-v0-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-sql-v0 + udf-service-exception_policy + library-yql-sql + yql-sql-pg_dummy +) +target_link_options(ydb-library-yql-sql-v0-ut PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-library-yql-sql-v0-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v0/sql_ut.cpp +) +set_property( + TARGET + ydb-library-yql-sql-v0-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-sql-v0-ut + TEST_TARGET + ydb-library-yql-sql-v0-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v0-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v0-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v0-ut + PROPERTY + TIMEOUT + 300 +) +target_allocator(ydb-library-yql-sql-v0-ut + system_allocator +) +vcs_info(ydb-library-yql-sql-v0-ut) diff --git a/ydb/library/yql/sql/v0/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/sql/v0/ut/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..69099ea1200 --- /dev/null +++ b/ydb/library/yql/sql/v0/ut/CMakeLists.linux-aarch64.txt @@ -0,0 +1,80 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-sql-v0-ut) +target_include_directories(ydb-library-yql-sql-v0-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v0 +) +target_link_libraries(ydb-library-yql-sql-v0-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + yql-sql-v0 + udf-service-exception_policy + library-yql-sql + yql-sql-pg_dummy +) +target_link_options(ydb-library-yql-sql-v0-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-sql-v0-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v0/sql_ut.cpp +) +set_property( + TARGET + ydb-library-yql-sql-v0-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-sql-v0-ut + TEST_TARGET + ydb-library-yql-sql-v0-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v0-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v0-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v0-ut + PROPERTY + TIMEOUT + 300 +) +target_allocator(ydb-library-yql-sql-v0-ut + cpp-malloc-jemalloc +) +vcs_info(ydb-library-yql-sql-v0-ut) diff --git a/ydb/library/yql/sql/v0/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/sql/v0/ut/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..d39e61187c3 --- /dev/null +++ b/ydb/library/yql/sql/v0/ut/CMakeLists.linux-x86_64.txt @@ -0,0 +1,82 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-sql-v0-ut) +target_include_directories(ydb-library-yql-sql-v0-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v0 +) +target_link_libraries(ydb-library-yql-sql-v0-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-sql-v0 + udf-service-exception_policy + library-yql-sql + yql-sql-pg_dummy +) +target_link_options(ydb-library-yql-sql-v0-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-sql-v0-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v0/sql_ut.cpp +) +set_property( + TARGET + ydb-library-yql-sql-v0-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-sql-v0-ut + TEST_TARGET + ydb-library-yql-sql-v0-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v0-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v0-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v0-ut + PROPERTY + TIMEOUT + 300 +) +target_allocator(ydb-library-yql-sql-v0-ut + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(ydb-library-yql-sql-v0-ut) diff --git a/ydb/library/yql/sql/v0/ut/CMakeLists.txt b/ydb/library/yql/sql/v0/ut/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/sql/v0/ut/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/sql/v0/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/sql/v0/ut/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..e8b26a38f79 --- /dev/null +++ b/ydb/library/yql/sql/v0/ut/CMakeLists.windows-x86_64.txt @@ -0,0 +1,70 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-sql-v0-ut) +target_include_directories(ydb-library-yql-sql-v0-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v0 +) +target_link_libraries(ydb-library-yql-sql-v0-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-sql-v0 + udf-service-exception_policy + library-yql-sql + yql-sql-pg_dummy +) +target_sources(ydb-library-yql-sql-v0-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v0/sql_ut.cpp +) +set_property( + TARGET + ydb-library-yql-sql-v0-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-sql-v0-ut + TEST_TARGET + ydb-library-yql-sql-v0-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v0-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v0-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v0-ut + PROPERTY + TIMEOUT + 300 +) +target_allocator(ydb-library-yql-sql-v0-ut + system_allocator +) +vcs_info(ydb-library-yql-sql-v0-ut) diff --git a/ydb/library/yql/sql/v0/ya.make b/ydb/library/yql/sql/v0/ya.make index 826ad664928..334083a6c90 100644 --- a/ydb/library/yql/sql/v0/ya.make +++ b/ydb/library/yql/sql/v0/ya.make @@ -33,6 +33,10 @@ GENERATE_ENUM_SERIALIZATION(node.h) END() +RECURSE( + lexer +) + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/library/yql/sql/v1/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/sql/v1/CMakeLists.darwin-x86_64.txt index 10523a71a47..6e0ae494e7d 100644 --- a/ydb/library/yql/sql/v1/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/sql/v1/CMakeLists.darwin-x86_64.txt @@ -10,6 +10,7 @@ add_subdirectory(format) add_subdirectory(lexer) add_subdirectory(perf) add_subdirectory(proto_parser) +add_subdirectory(ut) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/sql/v1/CMakeLists.linux-aarch64.txt b/ydb/library/yql/sql/v1/CMakeLists.linux-aarch64.txt index f01282c6873..17cb39be0ff 100644 --- a/ydb/library/yql/sql/v1/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/sql/v1/CMakeLists.linux-aarch64.txt @@ -10,6 +10,7 @@ add_subdirectory(format) add_subdirectory(lexer) add_subdirectory(perf) add_subdirectory(proto_parser) +add_subdirectory(ut) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/sql/v1/CMakeLists.linux-x86_64.txt b/ydb/library/yql/sql/v1/CMakeLists.linux-x86_64.txt index f01282c6873..17cb39be0ff 100644 --- a/ydb/library/yql/sql/v1/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/sql/v1/CMakeLists.linux-x86_64.txt @@ -10,6 +10,7 @@ add_subdirectory(format) add_subdirectory(lexer) add_subdirectory(perf) add_subdirectory(proto_parser) +add_subdirectory(ut) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/sql/v1/CMakeLists.windows-x86_64.txt b/ydb/library/yql/sql/v1/CMakeLists.windows-x86_64.txt index 10523a71a47..6e0ae494e7d 100644 --- a/ydb/library/yql/sql/v1/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/sql/v1/CMakeLists.windows-x86_64.txt @@ -10,6 +10,7 @@ add_subdirectory(format) add_subdirectory(lexer) add_subdirectory(perf) add_subdirectory(proto_parser) +add_subdirectory(ut) get_built_tool_path( TOOL_enum_parser_bin TOOL_enum_parser_dependency diff --git a/ydb/library/yql/sql/v1/format/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/sql/v1/format/CMakeLists.darwin-x86_64.txt index 0b8bfe0914a..f5b30fd770a 100644 --- a/ydb/library/yql/sql/v1/format/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/sql/v1/format/CMakeLists.darwin-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) get_built_tool_path( TOOL_rescompiler_bin TOOL_rescompiler_dependency diff --git a/ydb/library/yql/sql/v1/format/CMakeLists.linux-aarch64.txt b/ydb/library/yql/sql/v1/format/CMakeLists.linux-aarch64.txt index cb1ff977c68..866b33730a6 100644 --- a/ydb/library/yql/sql/v1/format/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/sql/v1/format/CMakeLists.linux-aarch64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) get_built_tool_path( TOOL_rescompiler_bin TOOL_rescompiler_dependency diff --git a/ydb/library/yql/sql/v1/format/CMakeLists.linux-x86_64.txt b/ydb/library/yql/sql/v1/format/CMakeLists.linux-x86_64.txt index cb1ff977c68..866b33730a6 100644 --- a/ydb/library/yql/sql/v1/format/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/sql/v1/format/CMakeLists.linux-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) get_built_tool_path( TOOL_rescompiler_bin TOOL_rescompiler_dependency diff --git a/ydb/library/yql/sql/v1/format/CMakeLists.windows-x86_64.txt b/ydb/library/yql/sql/v1/format/CMakeLists.windows-x86_64.txt index 0b8bfe0914a..f5b30fd770a 100644 --- a/ydb/library/yql/sql/v1/format/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/sql/v1/format/CMakeLists.windows-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) get_built_tool_path( TOOL_rescompiler_bin TOOL_rescompiler_dependency diff --git a/ydb/library/yql/sql/v1/format/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/sql/v1/format/ut/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..4a924acf871 --- /dev/null +++ b/ydb/library/yql/sql/v1/format/ut/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,67 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-sql-v1-format-ut) +target_include_directories(ydb-library-yql-sql-v1-format-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/format +) +target_link_libraries(ydb-library-yql-sql-v1-format-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + sql-v1-format +) +target_link_options(ydb-library-yql-sql-v1-format-ut PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-library-yql-sql-v1-format-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/format/sql_format_ut.cpp +) +set_property( + TARGET + ydb-library-yql-sql-v1-format-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-sql-v1-format-ut + TEST_TARGET + ydb-library-yql-sql-v1-format-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-format-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-format-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-sql-v1-format-ut + system_allocator +) +vcs_info(ydb-library-yql-sql-v1-format-ut) diff --git a/ydb/library/yql/sql/v1/format/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/sql/v1/format/ut/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..9b78b99e8c0 --- /dev/null +++ b/ydb/library/yql/sql/v1/format/ut/CMakeLists.linux-aarch64.txt @@ -0,0 +1,70 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-sql-v1-format-ut) +target_include_directories(ydb-library-yql-sql-v1-format-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/format +) +target_link_libraries(ydb-library-yql-sql-v1-format-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + sql-v1-format +) +target_link_options(ydb-library-yql-sql-v1-format-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-sql-v1-format-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/format/sql_format_ut.cpp +) +set_property( + TARGET + ydb-library-yql-sql-v1-format-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-sql-v1-format-ut + TEST_TARGET + ydb-library-yql-sql-v1-format-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-format-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-format-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-sql-v1-format-ut + cpp-malloc-jemalloc +) +vcs_info(ydb-library-yql-sql-v1-format-ut) diff --git a/ydb/library/yql/sql/v1/format/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/sql/v1/format/ut/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..d8fed6326eb --- /dev/null +++ b/ydb/library/yql/sql/v1/format/ut/CMakeLists.linux-x86_64.txt @@ -0,0 +1,72 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-sql-v1-format-ut) +target_include_directories(ydb-library-yql-sql-v1-format-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/format +) +target_link_libraries(ydb-library-yql-sql-v1-format-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + sql-v1-format +) +target_link_options(ydb-library-yql-sql-v1-format-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-sql-v1-format-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/format/sql_format_ut.cpp +) +set_property( + TARGET + ydb-library-yql-sql-v1-format-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-sql-v1-format-ut + TEST_TARGET + ydb-library-yql-sql-v1-format-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-format-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-format-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-sql-v1-format-ut + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(ydb-library-yql-sql-v1-format-ut) diff --git a/ydb/library/yql/sql/v1/format/ut/CMakeLists.txt b/ydb/library/yql/sql/v1/format/ut/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/sql/v1/format/ut/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/sql/v1/format/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/sql/v1/format/ut/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..c130d878505 --- /dev/null +++ b/ydb/library/yql/sql/v1/format/ut/CMakeLists.windows-x86_64.txt @@ -0,0 +1,60 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-sql-v1-format-ut) +target_include_directories(ydb-library-yql-sql-v1-format-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/format +) +target_link_libraries(ydb-library-yql-sql-v1-format-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + sql-v1-format +) +target_sources(ydb-library-yql-sql-v1-format-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/format/sql_format_ut.cpp +) +set_property( + TARGET + ydb-library-yql-sql-v1-format-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-sql-v1-format-ut + TEST_TARGET + ydb-library-yql-sql-v1-format-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-format-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-format-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-library-yql-sql-v1-format-ut + system_allocator +) +vcs_info(ydb-library-yql-sql-v1-format-ut) diff --git a/ydb/library/yql/sql/v1/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/sql/v1/ut/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..17379347b27 --- /dev/null +++ b/ydb/library/yql/sql/v1/ut/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,79 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-sql-v1-ut) +target_include_directories(ydb-library-yql-sql-v1-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1 +) +target_link_libraries(ydb-library-yql-sql-v1-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-sql-v1 + udf-service-exception_policy + library-yql-sql + yql-sql-pg_dummy + sql-v1-format +) +target_link_options(ydb-library-yql-sql-v1-ut PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-library-yql-sql-v1-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/sql_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/sql_match_recognize_ut.cpp +) +set_property( + TARGET + ydb-library-yql-sql-v1-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-sql-v1-ut + TEST_TARGET + ydb-library-yql-sql-v1-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-ut + PROPERTY + TIMEOUT + 300 +) +target_allocator(ydb-library-yql-sql-v1-ut + system_allocator +) +vcs_info(ydb-library-yql-sql-v1-ut) diff --git a/ydb/library/yql/sql/v1/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/sql/v1/ut/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..e752dcf4475 --- /dev/null +++ b/ydb/library/yql/sql/v1/ut/CMakeLists.linux-aarch64.txt @@ -0,0 +1,82 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-sql-v1-ut) +target_include_directories(ydb-library-yql-sql-v1-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1 +) +target_link_libraries(ydb-library-yql-sql-v1-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + yql-sql-v1 + udf-service-exception_policy + library-yql-sql + yql-sql-pg_dummy + sql-v1-format +) +target_link_options(ydb-library-yql-sql-v1-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-sql-v1-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/sql_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/sql_match_recognize_ut.cpp +) +set_property( + TARGET + ydb-library-yql-sql-v1-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-sql-v1-ut + TEST_TARGET + ydb-library-yql-sql-v1-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-ut + PROPERTY + TIMEOUT + 300 +) +target_allocator(ydb-library-yql-sql-v1-ut + cpp-malloc-jemalloc +) +vcs_info(ydb-library-yql-sql-v1-ut) diff --git a/ydb/library/yql/sql/v1/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/sql/v1/ut/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..cbe9969c009 --- /dev/null +++ b/ydb/library/yql/sql/v1/ut/CMakeLists.linux-x86_64.txt @@ -0,0 +1,84 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-sql-v1-ut) +target_include_directories(ydb-library-yql-sql-v1-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1 +) +target_link_libraries(ydb-library-yql-sql-v1-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-sql-v1 + udf-service-exception_policy + library-yql-sql + yql-sql-pg_dummy + sql-v1-format +) +target_link_options(ydb-library-yql-sql-v1-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-library-yql-sql-v1-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/sql_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/sql_match_recognize_ut.cpp +) +set_property( + TARGET + ydb-library-yql-sql-v1-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-sql-v1-ut + TEST_TARGET + ydb-library-yql-sql-v1-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-ut + PROPERTY + TIMEOUT + 300 +) +target_allocator(ydb-library-yql-sql-v1-ut + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(ydb-library-yql-sql-v1-ut) diff --git a/ydb/library/yql/sql/v1/ut/CMakeLists.txt b/ydb/library/yql/sql/v1/ut/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/ydb/library/yql/sql/v1/ut/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/sql/v1/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/sql/v1/ut/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..2352e1c720f --- /dev/null +++ b/ydb/library/yql/sql/v1/ut/CMakeLists.windows-x86_64.txt @@ -0,0 +1,72 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-library-yql-sql-v1-ut) +target_include_directories(ydb-library-yql-sql-v1-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1 +) +target_link_libraries(ydb-library-yql-sql-v1-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + yql-sql-v1 + udf-service-exception_policy + library-yql-sql + yql-sql-pg_dummy + sql-v1-format +) +target_sources(ydb-library-yql-sql-v1-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/sql_ut.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/sql_match_recognize_ut.cpp +) +set_property( + TARGET + ydb-library-yql-sql-v1-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-library-yql-sql-v1-ut + TEST_TARGET + ydb-library-yql-sql-v1-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-ut + PROPERTY + LABELS + MEDIUM +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-ut + PROPERTY + PROCESSORS + 1 +) +set_yunittest_property( + TEST + ydb-library-yql-sql-v1-ut + PROPERTY + TIMEOUT + 300 +) +target_allocator(ydb-library-yql-sql-v1-ut + system_allocator +) +vcs_info(ydb-library-yql-sql-v1-ut) diff --git a/ydb/library/yql/sql/v1/ya.make b/ydb/library/yql/sql/v1/ya.make index 5ecab38ae67..c6d812c707c 100644 --- a/ydb/library/yql/sql/v1/ya.make +++ b/ydb/library/yql/sql/v1/ya.make @@ -56,6 +56,13 @@ GENERATE_ENUM_SERIALIZATION(sql_call_param.h) END() +RECURSE( + format + lexer + perf + proto_parser +) + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/library/yql/sql/ya.make b/ydb/library/yql/sql/ya.make index 9d4ff1450c5..92837d28714 100644 --- a/ydb/library/yql/sql/ya.make +++ b/ydb/library/yql/sql/ya.make @@ -23,4 +23,7 @@ END() RECURSE( pg pg_dummy + settings + v0 + v1 ) diff --git a/ydb/library/yql/tools/mrjob/ya.make b/ydb/library/yql/tools/mrjob/ya.make index c91ff06cda8..95ae41ec65c 100644 --- a/ydb/library/yql/tools/mrjob/ya.make +++ b/ydb/library/yql/tools/mrjob/ya.make @@ -25,3 +25,7 @@ PEERDIR( YQL_LAST_ABI_VERSION() END() + +RECURSE_FOR_TESTS( + test +) diff --git a/ydb/library/yql/tools/ya.make b/ydb/library/yql/tools/ya.make index 062df88349a..3e928cff45d 100644 --- a/ydb/library/yql/tools/ya.make +++ b/ydb/library/yql/tools/ya.make @@ -1,6 +1,5 @@ RECURSE( astdiff mrjob - mrjob/test yqlrun ) diff --git a/ydb/library/yql/utils/backtrace/ya.make b/ydb/library/yql/utils/backtrace/ya.make index 93d53065807..15836994993 100644 --- a/ydb/library/yql/utils/backtrace/ya.make +++ b/ydb/library/yql/utils/backtrace/ya.make @@ -29,6 +29,10 @@ PEERDIR( END() +RECURSE( + fake_llvm_symbolizer +) + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/library/yql/utils/ya.make b/ydb/library/yql/utils/ya.make index f5ea18eb6c5..e8df4b5e156 100644 --- a/ydb/library/yql/utils/ya.make +++ b/ydb/library/yql/utils/ya.make @@ -57,6 +57,7 @@ RECURSE( failure_injector fetch log + test_http_server threading ) diff --git a/ydb/library/yql/ya.make b/ydb/library/yql/ya.make index 83b1b9e9270..13581288157 100644 --- a/ydb/library/yql/ya.make +++ b/ydb/library/yql/ya.make @@ -1,21 +1,6 @@ RECURSE( ast core - core/arrow_kernels/request - core/arrow_kernels/registry - core/common_opt - core/credentials - core/expr_nodes - core/expr_nodes_gen - core/extract_predicate - core/facade - core/file_storage - core/issue - core/peephole_opt - core/services - core/sql_types - core/type_ann - core/user_data dq minikql parser |